diff --git a/cmd/backup/main.go b/cmd/backup/main.go index eda6019..5941a30 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -44,7 +44,7 @@ func main() { }() s.must(s.withLabeledCommands(lifecyclePhaseArchive, func() error { - restartContainers, err := s.stopContainers() + restartContainers, err := s.stopContainersAndServices() // The mechanism for restarting containers is not using hooks as it // should happen as soon as possible (i.e. before uploading backups or // similar). diff --git a/cmd/backup/script.go b/cmd/backup/script.go index 1bed6a7..3d66aac 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -318,44 +318,59 @@ func newScript() (*script, error) { return s, nil } -// stopContainers stops all Docker containers that are marked as to being +// stopContainersAndServices stops all Docker containers that are marked as to being // stopped during the backup and returns a function that can be called to // restart everything that has been stopped. -func (s *script) stopContainers() (func() error, error) { +func (s *script) stopContainersAndServices() (func() error, error) { if s.cli == nil { return noop, nil } + matchLabel := fmt.Sprintf( + "docker-volume-backup.stop-during-backup=%s", + s.c.BackupStopContainerLabel, + ) + allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{}) if err != nil { return noop, fmt.Errorf("stopContainers: error querying for containers: %w", err) } - - containerLabel := fmt.Sprintf( - "docker-volume-backup.stop-during-backup=%s", - s.c.BackupStopContainerLabel, - ) containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{ Filters: filters.NewArgs(filters.KeyValuePair{ Key: "label", - Value: containerLabel, + Value: matchLabel, }), }) - if err != nil { return noop, fmt.Errorf("stopContainers: error querying for containers to stop: %w", err) } - if len(containersToStop) == 0 { + allServices, err := s.cli.ServiceList(context.Background(), types.ServiceListOptions{}) + if err != nil { + return noop, fmt.Errorf("stopContainers: error querying for services: %w", err) + } + servicesToScaleDown, err := s.cli.ServiceList(context.Background(), types.ServiceListOptions{ + Filters: filters.NewArgs(filters.KeyValuePair{ + Key: "label", + Value: matchLabel, + }), + }) + if err != nil { + return noop, fmt.Errorf("stopContainers: error querying for services to scale down: %w", err) + } + + if len(containersToStop) == 0 && len(servicesToScaleDown) == 0 { return noop, nil } s.logger.Info( fmt.Sprintf( - "Stopping %d container(s) labeled `%s` out of %d running container(s).", + "Stopping %d container(s) out of %d running container(s) and scaling down %d service(s) out of %d, as they were labeled %s.", len(containersToStop), - containerLabel, len(allContainers), + len(servicesToScaleDown), + len(allServices), + matchLabel, ), ) @@ -385,12 +400,12 @@ func (s *script) stopContainers() (func() error, error) { } return func() error { - servicesRequiringUpdate := map[string]struct{}{} + servicesRequiringForceUpdate := map[string]struct{}{} var restartErrors []error for _, container := range stoppedContainers { if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok { - servicesRequiringUpdate[swarmServiceName] = struct{}{} + servicesRequiringForceUpdate[swarmServiceName] = struct{}{} continue } if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil { @@ -398,9 +413,9 @@ func (s *script) stopContainers() (func() error, error) { } } - if len(servicesRequiringUpdate) != 0 { + if len(servicesRequiringForceUpdate) != 0 { services, _ := s.cli.ServiceList(context.Background(), types.ServiceListOptions{}) - for serviceName := range servicesRequiringUpdate { + for serviceName := range servicesRequiringForceUpdate { var serviceMatch swarm.Service for _, service := range services { if service.Spec.Name == serviceName { diff --git a/test/services/docker-compose.yml b/test/services/docker-compose.yml new file mode 100644 index 0000000..dbbf8b9 --- /dev/null +++ b/test/services/docker-compose.yml @@ -0,0 +1,69 @@ +# Copyright 2020-2021 - Offen Authors +# SPDX-License-Identifier: Unlicense + +version: '3.8' + +services: + minio: + image: minio/minio:RELEASE.2020-08-04T23-10-51Z + deploy: + restart_policy: + condition: on-failure + environment: + MINIO_ROOT_USER: test + MINIO_ROOT_PASSWORD: test + MINIO_ACCESS_KEY: test + MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ + entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data' + volumes: + - backup_data:/data + + backup: + image: offen/docker-volume-backup:${TEST_VERSION:-canary} + depends_on: + - minio + deploy: + restart_policy: + condition: on-failure + environment: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ + AWS_ENDPOINT: minio:9000 + AWS_ENDPOINT_PROTO: http + AWS_S3_BUCKET_NAME: backup + BACKUP_FILENAME: test.tar.gz + BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? + BACKUP_RETENTION_DAYS: 7 + BACKUP_PRUNING_LEEWAY: 5s + volumes: + - pg_data:/backup/pg_data:ro + - /var/run/docker.sock:/var/run/docker.sock + + offen: + image: offen/offen:latest + healthcheck: + disable: true + deploy: + labels: + - docker-volume-backup.stop-during-backup=true + replicas: 2 + restart_policy: + condition: on-failure + + pg: + image: postgres:14-alpine + environment: + POSTGRES_PASSWORD: example + labels: + - docker-volume-backup.stop-during-backup=true + volumes: + - pg_data:/var/lib/postgresql/data + deploy: + restart_policy: + condition: on-failure + +volumes: + backup_data: + name: backup_data + pg_data: + name: pg_data diff --git a/test/services/run.sh b/test/services/run.sh new file mode 100755 index 0000000..ad1f8c0 --- /dev/null +++ b/test/services/run.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e + +cd $(dirname $0) +. ../util.sh +current_test=$(basename $(pwd)) + +docker swarm init + +docker stack deploy --compose-file=docker-compose.yml test_stack + +while [ -z $(docker ps -q -f name=backup) ]; do + info "Backup container not ready yet. Retrying." + sleep 1 +done + +sleep 20 + +docker exec $(docker ps -q -f name=backup) backup + +docker run --rm \ + -v backup_data:/data alpine \ + ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION' + +pass "Found relevant files in untared backup." + +sleep 5 +expect_running_containers "5"