Run tests Docker in Docker (#261)

* Try running tests in Docker

* Spawn new container for each test

* Store test artifacts outside of mount

* When requested, build up to date image in test script

* sudo is unneccessary in containerized test env

* Skip azure test

* Backdate fixture file in JSON database

* Pin versions for azure tools

* Mount temp volume for /var/lib/docker to prevent dangling ones created by VOLUME instruction

* Fail backdating tests with message

* Add some documentation on test setup

* Cache images

* Run compose stacks with shortened default timeout

.github/workflows/test.yml

@@ -15,16 +15,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Build Docker Image
-        env:
-          DOCKER_BUILDKIT: '1'
-        run: docker build . -t offen/docker-volume-backup:test
-
       - name: Run Tests
         working-directory: ./test
         run: |
-          # Stop the buildx container so the tests can make assertions
+          BUILD_IMAGE=1 ./test.sh
-          # about the number of running containers
-          docker rm -f $(docker ps -aq)
-          export GPG_TTY=$(tty)
-          ./test.sh test

test/Dockerfile

@@ -0,0 +1,13 @@
+FROM docker:24-dind
+
+RUN apk add \
+  coreutils \
+  curl \
+  gpg \
+  jq \
+  moreutils \
+  tar \
+  zstd \
+  --no-cache
+
+WORKDIR /code/test

test/README.md

@@ -0,0 +1,70 @@
+# Integration Tests
+
+## Running tests
+
+The main entry point for running tests is the `./test.sh` script.
+It can be used to run the entire test suite, or just a single test case.
+
+### Run all tests
+
+```sh
+./test.sh
+```
+
+### Run a single test case
+
+```sh
+./test.sh <directory-name>
+```
+
+### Configuring a test run
+
+In addition to the match pattern, which can be given as the first positional argument, certain behavior can be changed by setting environment variables:
+
+#### `BUILD_IMAGE`
+
+When set, the test script will build an up-to-date `docker-volume-backup` image from the current state of your source tree, and run the tests against it.
+
+```sh
+BUILD_IMAGE=1 ./test.sh
+```
+
+The default behavior is not to build an image, and instead look for a version on your host system.
+
+#### `IMAGE_TAG`
+
+Setting this value lets you run tests against different existing images, so you can compare behavior:
+
+```sh
+IMAGE_TAG=v2.30.0 ./test.sh
+```
+
+#### `NO_IMAGE_CACHE`
+
+When set, images from remote registries will not be cached and shared between sandbox containers.
+
+```sh
+NO_IMAGE_CACHE=1 ./test.sh
+```
+
+By default, two local images are created that persist the image data and provide it to containers at runtime.
+
+## Understanding the test setup
+
+The test setup runs each test case in an isolated Docker container, which itself is running an otherwise unused Docker daemon.
+This means, tests can rely on noone else using that daemon, making expectations about the number of running containers and so forth.
+As the sandbox container is also expected to be torn down post test, the scripts do not need to do any clean up or similar.
+
+## Anatomy of a test case
+
+The `test.sh` script looks for an exectuable file called `run.sh` in each directory.
+When found, it is executed and signals success by returning a 0 exit code.
+Any other exit code is considered a failure and will halt execution of further tests.
+
+There is an `util.sh` file containing a few commonly used helpers which can be used by putting the following prelude to a new test case:
+
+```sh
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+```

test/azure/docker-compose.yml

@@ -2,19 +2,19 @@ version: '3'
 
 services:
   storage:
-    image: mcr.microsoft.com/azure-storage/azurite
+    image: mcr.microsoft.com/azure-storage/azurite:3.26.0
     volumes:
-      - azurite_backup_data:/data
+      - ${DATA_DIR:-./data}:/data
     command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data
     healthcheck:
-        test: nc 127.0.0.1 10000 -z
+      test: nc 127.0.0.1 10000 -z
-        interval: 1s
+      interval: 1s
-        retries: 30
+      retries: 30
 
   az_cli:
-    image: mcr.microsoft.com/azure-cli
+    image: mcr.microsoft.com/azure-cli:2.51.0
     volumes:
-      - ./local:/dump
+      - ${LOCAL_DIR:-./local}:/dump
     command:
       - /bin/sh
       - -c
@@ -53,6 +53,4 @@ services:
       - app_data:/var/opt/offen
 
 volumes:
-  azurite_backup_data:
-    name: azurite_backup_data
   app_data:

test/azure/run.sh

@@ -6,12 +6,17 @@ cd "$(dirname "$0")"
 . ../util.sh
 current_test=$(basename $(pwd))
 
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
+export DATA_DIR=$(mktemp -d)
+
 download_az () {
   docker compose run --rm az_cli \
     az storage blob download -f /dump/$1.tar.gz -c test-container -n path/to/backup/$1.tar.gz
 }
 
 docker compose up -d --quiet-pull
+
 sleep 5
 
 docker compose exec backup backup
@@ -21,9 +26,15 @@ sleep 5
 expect_running_containers "3"
 
 download_az "test"
-tar -xvf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
+
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR
+
+if [ ! -f "$TMP_DIR/backup/app_data/offen.db" ]; then
+  fail "Could not find expeced file in untared backup"
+fi
 
 pass "Found relevant files in untared remote backups."
+rm "$LOCAL_DIR/test.tar.gz"
 
 # The second part of this test checks if backups get deleted when the retention
 # is set to 0 days (which it should not as it would mean all backups get deleted)
@@ -33,8 +44,9 @@ sleep 5
 docker compose exec backup backup
 
 download_az "test"
-test -f ./local/test.tar.gz
+if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then
-
+  fail "Remote backup was deleted"
+fi
 pass "Remote backups have not been deleted."
 
 # The third part of this test checks if old backups get deleted when the retention
@@ -46,21 +58,29 @@ sleep 5
 info "Create first backup with no prune"
 docker compose exec backup backup
 
-sudo date --set="14 days ago"
-
 docker compose run --rm az_cli \
     az storage blob upload -f /dump/test.tar.gz -c test-container -n path/to/backup/test-old.tar.gz
 
-sudo date --set="14 days"
+docker compose down
+rm "$LOCAL_DIR/test.tar.gz"
+
+back_date="$(date "+%Y-%m-%dT%H:%M:%S%z" -d "14 days ago" | rev | cut -c 3- | rev):00"
+jq --arg back_date "$back_date" '(.collections[] | select(.name=="$BLOBS_COLLECTION$") | .data[] | select(.name=="path/to/backup/test-old.tar.gz") | .properties.creationTime = $back_date)' "$DATA_DIR/__azurite_db_blob__.json" | sponge "$DATA_DIR/__azurite_db_blob__.json"
+
+docker compose up -d
+sleep 5
 
 info "Create second backup and prune"
 docker compose exec backup backup
 
 info "Download first backup which should be pruned"
 download_az "test-old" || true
-test ! -f ./local/test-old.tar.gz
+if [ -f "$LOCAL_DIR/test-old.tar.gz" ]; then
-test -f ./local/test.tar.gz
+  fail "Backdated file was not deleted"
+fi
+download_az "test" || true
+if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then
+  fail "Recent file was not found"
+fi
 
 pass "Old remote backup has been pruned, new one is still present."
-
-docker compose down --volumes

test/certs/docker-compose.yml

@@ -12,8 +12,8 @@ services:
     entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server --certs-dir "/certs" --address ":443" /data'
     volumes:
       - minio_backup_data:/data
-      - ./minio.crt:/certs/public.crt
+      - ${CERT_DIR:-.}/minio.crt:/certs/public.crt
-      - ./minio.key:/certs/private.key
+      - ${CERT_DIR:-.}/minio.key:/certs/private.key
 
   backup:
     image: offen/docker-volume-backup:${TEST_VERSION:-canary}
@@ -33,7 +33,7 @@ services:
     volumes:
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
-      - ./rootCA.crt:/root/minio-rootCA.crt
+      - ${CERT_DIR:-.}/rootCA.crt:/root/minio-rootCA.crt
 
   offen:
     image: offen/offen:latest

test/certs/run.sh

@@ -6,23 +6,25 @@ cd "$(dirname "$0")"
 . ../util.sh
 current_test=$(basename $(pwd))
 
-openssl genrsa -des3 -passout pass:test -out rootCA.key 4096
+export CERT_DIR=$(mktemp -d)
+
+openssl genrsa -des3 -passout pass:test -out "$CERT_DIR/rootCA.key" 4096
 openssl req -passin pass:test \
   -subj "/C=DE/ST=BE/O=IntegrationTest, Inc." \
-  -x509 -new -key rootCA.key -sha256 -days 1 -out rootCA.crt
+  -x509 -new -key "$CERT_DIR/rootCA.key" -sha256 -days 1 -out "$CERT_DIR/rootCA.crt"
 
-openssl genrsa -out minio.key 4096
+openssl genrsa -out "$CERT_DIR/minio.key" 4096
-openssl req -new -sha256 -key minio.key \
+openssl req -new -sha256 -key "$CERT_DIR/minio.key" \
   -subj "/C=DE/ST=BE/O=IntegrationTest, Inc./CN=minio" \
-  -out minio.csr
+  -out "$CERT_DIR/minio.csr"
 
 openssl x509 -req -passin pass:test \
-  -in minio.csr \
+  -in "$CERT_DIR/minio.csr" \
-  -CA rootCA.crt -CAkey rootCA.key -CAcreateserial \
+  -CA "$CERT_DIR/rootCA.crt" -CAkey "$CERT_DIR/rootCA.key" -CAcreateserial \
   -extfile san.cnf \
-  -out minio.crt -days 1 -sha256
+  -out "$CERT_DIR/minio.crt" -days 1 -sha256
 
-openssl x509 -in minio.crt -noout -text
+openssl x509 -in "$CERT_DIR/minio.crt" -noout -text
 
 docker compose up -d --quiet-pull
 sleep 5
@@ -39,5 +41,3 @@ docker run --rm \
   ash -c 'tar -xvf /minio_data/backup/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
 
 pass "Found relevant files in untared remote backups."
-
-docker compose down --volumes

test/commands/.gitignore

@@ -1 +0,0 @@
-local

test/commands/docker-compose.yml

@@ -42,7 +42,7 @@ services:
       EXEC_LABEL: test
       EXEC_FORWARD_OUTPUT: "true"
     volumes:
-      - ./local:/archive
+      - ${LOCAL_DIR:-./local}:/archive
       - app_data:/backup/data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 

test/commands/run.sh

@@ -6,36 +6,37 @@ cd $(dirname $0)
 . ../util.sh
 current_test=$(basename $(pwd))
 
-mkdir -p ./local
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
 
 docker compose up -d --quiet-pull
 sleep 30 # mariadb likes to take a bit before responding
 
 docker compose exec backup backup
 
-tar -xvf ./local/test.tar.gz
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR
-if [ ! -f ./backup/data/dump.sql ]; then
+if [ ! -f "$TMP_DIR/backup/data/dump.sql" ]; then
   fail "Could not find file written by pre command."
 fi
 pass "Found expected file."
 
-if [ -f ./backup/data/not-relevant.txt ]; then
+if [ -f "$TMP_DIR/backup/data/not-relevant.txt" ]; then
   fail "Command ran for container with other label."
 fi
 pass "Command did not run for container with other label."
 
-if [ -f ./backup/data/post.txt ]; then
+if [ -f "$TMP_DIR/backup/data/post.txt" ]; then
   fail "File created in post command was present in backup."
 fi
 pass "Did not find unexpected file."
 
 docker compose down --volumes
-sudo rm -rf ./local
-
 
 info "Running commands test in swarm mode next."
 
-mkdir -p ./local
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
+
 docker swarm init
 
 docker stack deploy --compose-file=docker-compose.yml test_stack
@@ -49,16 +50,13 @@ sleep 20
 
 docker exec $(docker ps -q -f name=backup) backup
 
-tar -xvf ./local/test.tar.gz
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR
-if [ ! -f ./backup/data/dump.sql ]; then
+if [ ! -f "$TMP_DIR/backup/data/dump.sql" ]; then
   fail "Could not find file written by pre command."
 fi
 pass "Found expected file."
 
-if [ -f ./backup/data/post.txt ]; then
+if [ -f "$TMP_DIR/backup/data/post.txt" ]; then
   fail "File created in post command was present in backup."
 fi
 pass "Did not find unexpected file."
-
-docker stack rm test_stack
-docker swarm leave --force

test/confd/.gitignore

@@ -1 +0,0 @@
-local

test/confd/docker-compose.yml

@@ -5,7 +5,7 @@ services:
     image: offen/docker-volume-backup:${TEST_VERSION:-canary}
     restart: always
     volumes:
-      - ./local:/archive
+      - ${LOCAL_DIR:-./local}:/archive
       - app_data:/backup/app_data:ro
       - ./01backup.env:/etc/dockervolumebackup/conf.d/01backup.env
       - ./02backup.env:/etc/dockervolumebackup/conf.d/02backup.env

test/confd/run.sh

@@ -6,26 +6,24 @@ cd $(dirname $0)
 . ../util.sh
 current_test=$(basename $(pwd))
 
-mkdir -p local
+export LOCAL_DIR=$(mktemp -d)
 
 docker compose up -d --quiet-pull
 
 # sleep until a backup is guaranteed to have happened on the 1 minute schedule
 sleep 100
 
-docker compose down --volumes
+if [ ! -f "$LOCAL_DIR/conf.tar.gz" ]; then
-
-if [ ! -f ./local/conf.tar.gz ]; then
   fail "Config from file was not used."
 fi
 pass "Config from file was used."
 
-if [ ! -f ./local/other.tar.gz ]; then
+if [ ! -f "$LOCAL_DIR/other.tar.gz" ]; then
   fail "Run on same schedule did not succeed."
 fi
 pass "Run on same schedule succeeded."
 
-if [ -f ./local/never.tar.gz ]; then
+if [ -f "$LOCAL_DIR/never.tar.gz" ]; then
   fail "Unexpected file was found."
 fi
 pass "Unexpected cron did not run."

test/dropbox/docker-compose.yml

@@ -9,7 +9,7 @@ services:
     ports:
       - 8080:8080
     volumes:
-      - ./user_v2_ready.yaml:/etc/openapi/user_v2.yaml
+      - ${SPEC_FILE:-./user_v2.yaml}:/etc/openapi/user_v2.yaml
 
   oauth2_mock:
     image: ghcr.io/navikt/mock-oauth2-server:1.0.0

test/dropbox/run.sh

@@ -6,9 +6,10 @@ cd "$(dirname "$0")"
 . ../util.sh
 current_test=$(basename $(pwd))
 
-cp user_v2.yaml user_v2_ready.yaml
+export SPEC_FILE=$(mktemp -d)/user_v2.yaml
-sudo sed -i 's/SERVER_MODIFIED_1/'"$(date "+%Y-%m-%dT%H:%M:%SZ")/g" user_v2_ready.yaml
+cp user_v2.yaml $SPEC_FILE
-sudo sed -i 's/SERVER_MODIFIED_2/'"$(date "+%Y-%m-%dT%H:%M:%SZ" -d "14 days ago")/g" user_v2_ready.yaml
+sed -i 's/SERVER_MODIFIED_1/'"$(date "+%Y-%m-%dT%H:%M:%SZ")/g" $SPEC_FILE
+sed -i 's/SERVER_MODIFIED_2/'"$(date "+%Y-%m-%dT%H:%M:%SZ" -d "14 days ago")/g" $SPEC_FILE
 
 docker compose up -d --quiet-pull
 sleep 5
@@ -42,7 +43,6 @@ fi
 
 # The third part of this test checks if old backups get deleted when the retention
 # is set to 7 days (which it should)
-
 BACKUP_RETENTION_DAYS="7" docker compose up -d
 sleep 5
 
@@ -59,7 +59,3 @@ elif echo "$logs" | grep -q "None of 1 existing backups were pruned"; then
 else
   fail "Pruning failed, unknown result: $logs"
 fi
-
-
-docker compose down --volumes
-rm user_v2_ready.yaml

test/extend/docker-compose.yml

@@ -11,7 +11,7 @@ services:
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       EXEC_FORWARD_OUTPUT: "true"
     volumes:
-      - ./local:/local
+      - ${LOCAL_DIR:-local}:/local
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 

test/extend/run.sh

@@ -6,7 +6,7 @@ cd "$(dirname "$0")"
 . ../util.sh
 current_test=$(basename $(pwd))
 
-mkdir -p local
+export LOCAL_DIR=$(mktemp -d)
 
 export BASE_VERSION="${TEST_VERSION:-canary}"
 export TEST_VERSION="${TEST_VERSION:-canary}-with-rsync"
@@ -22,8 +22,6 @@ sleep 5
 
 expect_running_containers "2"
 
-if [ ! -f "./local/app_data/offen.db" ]; then
+if [ ! -f "$LOCAL_DIR/app_data/offen.db" ]; then
   fail "Could not find expected file in untared archive."
 fi
-
-docker compose down --volumes

test/gpg/.gitignore

@@ -1 +0,0 @@
-local

test/gpg/docker-compose.yml

@@ -11,7 +11,7 @@ services:
       BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
       GPG_PASSPHRASE: 1234#$$ecret
     volumes:
-      - ./local:/archive
+      - ${LOCAL_DIR:-./local}:/archive
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 

test/gpg/run.sh

@@ -6,7 +6,7 @@ cd "$(dirname "$0")"
 . ../util.sh
 current_test=$(basename $(pwd))
 
-mkdir -p local
+export LOCAL_DIR=$(mktemp -d)
 
 docker compose up -d --quiet-pull
 sleep 5
@@ -15,19 +15,18 @@ docker compose exec backup backup
 
 expect_running_containers "2"
 
-tmp_dir=$(mktemp -d)
+TMP_DIR=$(mktemp -d)
 
-echo "1234#\$ecret" | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 ./local/test.tar.gz.gpg > ./local/decrypted.tar.gz
+echo "1234#\$ecret" | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 "$LOCAL_DIR/test.tar.gz.gpg" > "$LOCAL_DIR/decrypted.tar.gz"
-tar -xf ./local/decrypted.tar.gz -C $tmp_dir
+tar -xf "$LOCAL_DIR/decrypted.tar.gz" -C $TMP_DIR
-if [ ! -f $tmp_dir/backup/app_data/offen.db ]; then
+
+if [ ! -f $TMP_DIR/backup/app_data/offen.db ]; then
   fail "Could not find expected file in untared archive."
 fi
-rm ./local/decrypted.tar.gz
+rm "$LOCAL_DIR/decrypted.tar.gz"
 
 pass "Found relevant files in decrypted and untared local backup."
 
-if [ ! -L ./local/test-latest.tar.gz.gpg ]; then
+if [ ! -L "$LOCAL_DIR/test-latest.tar.gz.gpg" ]; then
   fail "Could not find local symlink to latest encrypted backup."
 fi
-
-docker compose down --volumes

test/ignore/.gitignore

@@ -1 +0,0 @@
-local

test/ignore/docker-compose.yml

@@ -11,5 +11,5 @@ services:
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_EXCLUDE_REGEXP: '\.(me|you)$$'
     volumes:
-      - ./local:/archive
+      - ${LOCAL_DIR:-./local}:/archive
       - ./sources:/backup/data:ro

test/ignore/run.sh

@@ -6,23 +6,21 @@ cd $(dirname $0)
 . ../util.sh
 current_test=$(basename $(pwd))
 
-mkdir -p local
+export LOCAL_DIR=$(mktemp -d)
 
 docker compose up -d --quiet-pull
 sleep 5
 docker compose exec backup backup
 
-docker compose down --volumes
+TMP_DIR=$(mktemp -d)
+tar --same-owner -xvf "$LOCAL_DIR/test.tar.gz" -C "$TMP_DIR"
 
-out=$(mktemp -d)
+if [ ! -f "$TMP_DIR/backup/data/me.txt" ]; then
-sudo tar --same-owner -xvf ./local/test.tar.gz -C "$out"
-
-if [ ! -f "$out/backup/data/me.txt" ]; then
   fail "Expected file was not found."
 fi
 pass "Expected file was found."
 
-if [ -f "$out/backup/data/skip.me" ]; then
+if [ -f "$TMP_DIR/backup/data/skip.me" ]; then
   fail "Ignored file was found."
 fi
 pass "Ignored file was not found."

test/local/.gitignore

@@ -1 +0,0 @@
-local

test/local/docker-compose.yml

@@ -16,7 +16,7 @@ services:
     volumes:
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
-      - ./local:/archive
+      - ${LOCAL_DIR:-./local}:/archive
 
   offen:
     image: offen/offen:latest

test/local/run.sh

@@ -6,7 +6,7 @@ cd "$(dirname "$0")"
 . ../util.sh
 current_test=$(basename $(pwd))
 
-mkdir -p local
+export LOCAL_DIR=$(mktemp -d)
 
 docker compose up -d --quiet-pull
 sleep 5
@@ -21,11 +21,11 @@ sleep 5
 expect_running_containers "2"
 
 tmp_dir=$(mktemp -d)
-tar -xvf ./local/test-hostnametoken.tar.gz -C $tmp_dir
+tar -xvf "$LOCAL_DIR/test-hostnametoken.tar.gz" -C $tmp_dir
 if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then
   fail "Could not find expected file in untared archive."
 fi
-rm -f ./local/test-hostnametoken.tar.gz
+rm -f "$LOCAL_DIR/test-hostnametoken.tar.gz"
 
 if [ ! -L "$tmp_dir/backup/app_data/db.link" ]; then
   fail "Could not find expected symlink in untared archive."
@@ -33,7 +33,7 @@ fi
 
 pass "Found relevant files in decrypted and untared local backup."
 
-if [ ! -L ./local/test-hostnametoken.latest.tar.gz.gpg ]; then
+if [ ! -L "$LOCAL_DIR/test-hostnametoken.latest.tar.gz.gpg" ]; then
   fail "Could not find symlink to latest version."
 fi
 
@@ -46,8 +46,8 @@ sleep 5
 
 docker compose exec backup backup
 
-if [ "$(find ./local -type f | wc -l)" != "1" ]; then
+if [ "$(find "$LOCAL_DIR" -type f | wc -l)" != "1" ]; then
-  fail "Backups should not have been deleted, instead seen: "$(find ./local -type f)""
+  fail "Backups should not have been deleted, instead seen: "$(find "$local_dir" -type f)""
 fi
 pass "Local backups have not been deleted."
 
@@ -60,14 +60,17 @@ sleep 5
 info "Create first backup with no prune"
 docker compose exec backup backup
 
-touch -r ./local/test-hostnametoken.tar.gz -d "14 days ago" ./local/test-hostnametoken-old.tar.gz
+touch -r "$LOCAL_DIR/test-hostnametoken.tar.gz" -d "14 days ago" "$LOCAL_DIR/test-hostnametoken-old.tar.gz"
 
 info "Create second backup and prune"
 docker compose exec backup backup
 
-test ! -f ./local/test-hostnametoken-old.tar.gz
+if [ -f "$LOCAL_DIR/test-hostnametoken-old.tar.gz" ]; then
-test -f ./local/test-hostnametoken.tar.gz
+  fail "Backdated file has not been deleted."
+fi
+
+if [ ! -f "$LOCAL_DIR/test-hostnametoken.tar.gz" ]; then
+  fail "Recent file has been deleted."
+fi
 
 pass "Old remote backup has been pruned, new one is still present."
-
-docker compose down --volumes

test/notifications/.gitignore

@@ -1 +0,0 @@
-local

test/notifications/docker-compose.yml

@@ -12,7 +12,7 @@ services:
       NOTIFICATION_URLS: ${NOTIFICATION_URLS}
       EXTRA_VALUE: extra-value
     volumes:
-      - ./local:/archive
+      - ${LOCAL_DIR:-./local}:/archive
       - app_data:/backup/app_data:ro
       - ./notifications.tmpl:/etc/dockervolumebackup/notifications.d/notifications.tmpl
 

test/notifications/run.sh

@@ -6,7 +6,7 @@ cd $(dirname $0)
 . ../util.sh
 current_test=$(basename $(pwd))
 
-mkdir -p local
+export LOCAL_DIR=$(mktemp -d)
 
 docker compose up -d --quiet-pull
 sleep 5
@@ -46,5 +46,3 @@ if [ "$MESSAGE_BODY" != "Backing up /tmp/test.tar.gz succeeded." ]; then
   fail "Unexpected notification body $MESSAGE_BODY"
 fi
 pass "Custom notification body was used."
-
-docker compose down --volumes

test/ownership/.gitignore

@@ -1 +0,0 @@
-local

test/ownership/docker-compose.yml

@@ -9,9 +9,9 @@ services:
     volumes:
       - postgres_data:/var/lib/postgresql/data
     environment:
-      - POSTGRES_PASSWORD=1FHJMSwt0yhIN1zS7I4DilGUhThBKq0x
+      POSTGRES_PASSWORD: 1FHJMSwt0yhIN1zS7I4DilGUhThBKq0x
-      - POSTGRES_USER=test
+      POSTGRES_USER: test
-      - POSTGRES_DB=test
+      POSTGRES_DB: test
 
   backup:
     image: offen/docker-volume-backup:${TEST_VERSION}
@@ -21,7 +21,7 @@ services:
     volumes:
       - postgres_data:/backup/postgres:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
-      - ./local:/archive
+      - ${LOCAL_DIR:-./local}:/archive
 
 volumes:
   postgres_data:

test/ownership/run.sh

@@ -7,24 +7,22 @@ cd $(dirname $0)
 . ../util.sh
 current_test=$(basename $(pwd))
 
-mkdir -p local
+export LOCAL_DIR=$(mktemp -d)
 
 docker compose up -d --quiet-pull
 sleep 5
 
 docker compose exec backup backup
 
-tmp_dir=$(mktemp -d)
+TMP_DIR=$(mktemp -d)
-sudo tar --same-owner -xvf ./local/backup.tar.gz -C $tmp_dir
+tar --same-owner -xvf "$LOCAL_DIR/backup.tar.gz" -C $TMP_DIR
 
-sudo find $tmp_dir/backup/postgres > /dev/null
+find $TMP_DIR/backup/postgres > /dev/null
 pass "Backup contains files at expected location"
 
-for file in $(sudo find $tmp_dir/backup/postgres); do
+for file in $(find $TMP_DIR/backup/postgres); do
-  if [ "$(sudo stat -c '%u:%g' $file)" != "70:70" ]; then
+  if [ "$(stat -c '%u:%g' $file)" != "70:70" ]; then
-    fail "Unexpected file ownership for $file: $(sudo stat -c '%u:%g' $file)"
+    fail "Unexpected file ownership for $file: $(stat -c '%u:%g' $file)"
   fi
 done
 pass "All files and directories in backup preserved their ownership."
-
-docker compose down --volumes

test/pruning/run.sh

@@ -57,7 +57,9 @@ info "Create backup with no prune for both backends"
 docker compose exec -e BACKUP_SKIP_BACKENDS_FROM_PRUNE="s3,local" backup backup
 
 info "Check if old backup has NOT been pruned (local)"
-test -f ./local/test-hostnametoken-old.tar.gz
+if [ ! -f ./local/test-hostnametoken-old.tar.gz ]; then
+  fail "Backdated file has not been deleted"
+fi
 
 info "Check if old backup has NOT been pruned (s3)"
 docker run --rm \
@@ -66,5 +68,3 @@ docker run --rm \
   ash -c 'test -f /minio_data/backup/test-hostnametoken-old.tar.gz'
 
 pass "Skipped all backends while pruning."
-
-docker compose down --volumes

test/s3/run.sh

@@ -59,5 +59,3 @@ docker run --rm \
   ash -c 'test ! -f /minio_data/backup/test-hostnametoken-old.tar.gz && test -f /minio_data/backup/test-hostnametoken.tar.gz'
 
 pass "Old remote backup has been pruned, new one is still present."
-
-docker compose down --volumes

test/secrets/run.sh

@@ -40,15 +40,3 @@ docker exec -e AWS_ACCESS_KEY_ID_FILE=/tmp/nonexistant $(docker ps -q -f name=ba
   && fail "Backup should have failed due to non existing file env variable."
 
 pass "Backup failed due to non existing file env variable."
-
-docker stack rm test_stack
-
-docker secret rm minio_root_password
-docker secret rm minio_root_user
-
-docker swarm leave --force
-
-sleep 10
-
-docker volume rm backup_data
-docker volume rm pg_data

test/ssh/docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - PGID=1000
       - USER_NAME=test
     volumes:
-      - ./id_rsa.pub:/config/.ssh/authorized_keys
+      - ${KEY_DIR:-.}/id_rsa.pub:/config/.ssh/authorized_keys
       - ssh_backup_data:/tmp
 
   backup:
@@ -30,7 +30,7 @@ services:
       SSH_REMOTE_PATH: /tmp
       SSH_IDENTITY_PASSPHRASE: test1234
     volumes:
-      - ./id_rsa:/root/.ssh/id_rsa
+      - ${KEY_DIR:-.}/id_rsa:/root/.ssh/id_rsa
       - app_data:/backup/app_data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 

test/ssh/run.sh

@@ -6,7 +6,9 @@ cd "$(dirname "$0")"
 . ../util.sh
 current_test=$(basename $(pwd))
 
-ssh-keygen -t rsa -m pem -b 4096 -N "test1234" -f id_rsa -C "docker-volume-backup@local"
+export KEY_DIR=$(mktemp -d)
+
+ssh-keygen -t rsa -m pem -b 4096 -N "test1234" -f "$KEY_DIR/id_rsa" -C "docker-volume-backup@local"
 
 docker compose up -d --quiet-pull
 sleep 5
@@ -63,6 +65,3 @@ docker run --rm \
   ash -c 'test ! -f /ssh_data/test-hostnametoken-old.tar.gz && test -f /ssh_data/test-hostnametoken.tar.gz'
 
 pass "Old remote backup has been pruned, new one is still present."
-
-docker compose down --volumes
-rm -f id_rsa id_rsa.pub

test/swarm/run.sh

@@ -27,12 +27,3 @@ pass "Found relevant files in untared backup."
 
 sleep 5
 expect_running_containers "5"
-
-docker stack rm test_stack
-sleep 1
-docker swarm leave --force
-
-sleep 10
-
-docker volume rm backup_data
-docker volume rm pg_data

test/test.sh

@@ -2,15 +2,69 @@
 
 set -e
 
-TEST_VERSION=${1:-canary}
+MATCH_PATTERN=$1
+IMAGE_TAG=${IMAGE_TAG:-canary}
 
-for dir in $(ls -d -- */); do
+sandbox="docker_volume_backup_test_sandbox"
-  test="${dir}run.sh"
+tarball="$(mktemp -d)/image.tar.gz"
+
+trap finish EXIT INT TERM
+
+finish () {
+  rm -rf $(dirname $tarball)
+  if [ ! -z $(docker ps -aq --filter=name=$sandbox) ]; then
+    docker rm -f $(docker stop $sandbox)
+  fi
+  if [ ! -z $(docker volume ls -q --filter=name="^${sandbox}\$") ]; then
+    docker volume rm $sandbox
+  fi
+}
+
+docker build -t offen/docker-volume-backup:test-sandbox .
+
+if [ ! -z "$BUILD_IMAGE" ]; then
+  docker build -t offen/docker-volume-backup:$IMAGE_TAG $(dirname $(pwd))
+fi
+
+docker save offen/docker-volume-backup:$IMAGE_TAG -o $tarball
+
+find_args="-mindepth 1 -maxdepth 1 -type d"
+if [ ! -z "$MATCH_PATTERN" ]; then
+  find_args="$find_args -name $MATCH_PATTERN"
+fi
+
+for dir in $(find $find_args | sort); do
+  dir=$(echo $dir | cut -c 3-)
   echo "################################################"
-  echo "Now running $test"
+  echo "Now running ${dir}"
   echo "################################################"
   echo ""
-  TEST_VERSION=$TEST_VERSION /bin/sh $test
+
+  test="${dir}/run.sh"
+  docker_run_args="--name "$sandbox" --detach \
+    --privileged \
+    -v $(dirname $(pwd)):/code \
+    -v $tarball:/cache/image.tar.gz \
+    -v $sandbox:/var/lib/docker"
+
+  if [ -z "$NO_IMAGE_CACHE" ]; then
+    docker_run_args="$docker_run_args \
+    -v "${sandbox}_image":/var/lib/docker/image \
+    -v "${sandbox}_overlay2":/var/lib/docker/overlay2"
+  fi
+
+  docker run $docker_run_args offen/docker-volume-backup:test-sandbox
+
+  until docker exec $sandbox /bin/sh -c 'docker info' > /dev/null 2>&1; do
+    sleep 0.5
+  done
+  sleep 0.5
+
+  docker exec $sandbox /bin/sh -c "docker load -i /cache/image.tar.gz"
+  docker exec -e TEST_VERSION=$IMAGE_TAG $sandbox /bin/sh -c "/code/test/$test"
+
+  docker rm $(docker stop $sandbox)
+  docker volume rm $sandbox
   echo ""
   echo "$test passed"
   echo ""

test/user/.gitignore

@@ -1,2 +0,0 @@
-local
-backup

test/user/docker-compose.yml

@@ -10,7 +10,6 @@ services:
       - docker-volume-backup.archive-pre.user=testuser
       - docker-volume-backup.archive-pre=/bin/sh -c 'whoami > /tmp/whoami.txt'
 
-
   backup:
     image: offen/docker-volume-backup:${TEST_VERSION:-canary}
     deploy:
@@ -21,7 +20,7 @@ services:
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       EXEC_FORWARD_OUTPUT: "true"
     volumes:
-      - ./local:/archive
+      - ${LOCAL_DIR:-./local}:/archive
       - app_data:/backup/data:ro
       - /var/run/docker.sock:/var/run/docker.sock
 

test/user/run.sh

@@ -6,25 +6,25 @@ cd $(dirname $0)
 . ../util.sh
 current_test=$(basename $(pwd))
 
-docker compose up -d --quiet-pull
+export LOCAL_DIR=$(mktemp -d)
+export TMP_DIR=$(mktemp -d)
 
+echo "LOCAL_DIR $LOCAL_DIR"
+echo "TMP_DIR $TMP_DIR"
+
+docker compose up -d --quiet-pull
 user_name=testuser
 docker exec user-alpine-1 adduser --disabled-password "$user_name"
 
 docker compose exec backup backup
 
-tar -xvf ./local/test.tar.gz
+tar -xvf "$LOCAL_DIR/test.tar.gz" -C "$TMP_DIR"
-if [ ! -f ./backup/data/whoami.txt ]; then
+if [ ! -f "$TMP_DIR/backup/data/whoami.txt" ]; then
   fail "Could not find file written by pre command."
 fi
 pass "Found expected file."
 
-tar -xvf ./local/test.tar.gz
+if [ "$(cat $TMP_DIR/backup/data/whoami.txt)" != "$user_name" ]; then
-if [ "$(cat ./backup/data/whoami.txt)" != "$user_name" ]; then
   fail "Could not find expected user name."
 fi
 pass "Found expected user."
-
-docker compose down --volumes
-sudo rm -rf ./local
-

test/util.sh

@@ -15,9 +15,31 @@ fail () {
   exit 1
 }
 
+skip () {
+  echo "[test:${current_test:-none}:skip] "$1""
+  exit 0
+}
+
 expect_running_containers () {
   if [ "$(docker ps -q | wc -l)" != "$1" ]; then
     fail "Expected $1 containers to be running, instead seen: "$(docker ps -a | wc -l)""
   fi
   pass "$1 containers running."
 }
+
+docker() {
+  case $1 in
+    compose)
+      shift
+      case $1 in
+        up)
+          shift
+          command docker compose up --timeout 3 "$@";;
+        *)
+          command docker compose "$@";;
+      esac
+      ;;
+    *)
+      command docker "$@";;
+  esac
+}

test/webdav/run.sh

@@ -61,5 +61,3 @@ docker run --rm \
   ash -c 'test ! -f /webdav_data/data/my/new/path/test-hostnametoken-old.tar.gz && test -f /webdav_data/data/my/new/path/test-hostnametoken.tar.gz'
 
 pass "Old remote backup has been pruned, new one is still present."
-
-docker compose down --volumes

test/zstd/run.sh

@@ -9,7 +9,7 @@ current_test=$(basename $(pwd))
 docker network create test_network
 docker volume create app_data
 
-mkdir -p local
+LOCAL_DIR=$(mktemp -d)
 
 docker run -d -q \
   --name offen \
@@ -22,7 +22,7 @@ sleep 10
 docker run --rm -q \
   --network test_network \
   -v app_data:/backup/app_data \
-  -v ./local:/archive \
+  -v $LOCAL_DIR:/archive \
   -v /var/run/docker.sock:/var/run/docker.sock \
   --env BACKUP_COMPRESSION=zst \
   --env BACKUP_FILENAME='test.{{ .Extension }}' \
@@ -30,7 +30,7 @@ docker run --rm -q \
   offen/docker-volume-backup:${TEST_VERSION:-canary}
 
 tmp_dir=$(mktemp -d)
-tar -xvf ./local/test.tar.zst --zstd -C $tmp_dir
+tar -xvf "$LOCAL_DIR/test.tar.zst" --zstd -C $tmp_dir
 if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then
   fail "Could not find expected file in untared archive."
 fi
@@ -39,8 +39,3 @@ pass "Found relevant files in untared local backup."
 # This test does not stop containers during backup. This is happening on
 # purpose in order to cover this setup as well.
 expect_running_containers "1"
-
-docker rm $(docker stop offen)
-
-docker volume rm app_data
-docker network rm test_network