Query for labeled services as well

cmd/backup/main.go

@@ -44,7 +44,7 @@ func main() {
 	}()
 
 	s.must(s.withLabeledCommands(lifecyclePhaseArchive, func() error {
-		restartContainers, err := s.stopContainers()
+		restartContainers, err := s.stopContainersAndServices()
 		// The mechanism for restarting containers is not using hooks as it
 		// should happen as soon as possible (i.e. before uploading backups or
 		// similar).

cmd/backup/script.go

@@ -318,44 +318,59 @@ func newScript() (*script, error) {
 	return s, nil
 }
 
-// stopContainers stops all Docker containers that are marked as to being
+// stopContainersAndServices stops all Docker containers that are marked as to being
 // stopped during the backup and returns a function that can be called to
 // restart everything that has been stopped.
-func (s *script) stopContainers() (func() error, error) {
+func (s *script) stopContainersAndServices() (func() error, error) {
 	if s.cli == nil {
 		return noop, nil
 	}
 
+	matchLabel := fmt.Sprintf(
+		"docker-volume-backup.stop-during-backup=%s",
+		s.c.BackupStopContainerLabel,
+	)
+
 	allContainers, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{})
 	if err != nil {
 		return noop, fmt.Errorf("stopContainers: error querying for containers: %w", err)
 	}
-
-	containerLabel := fmt.Sprintf(
-		"docker-volume-backup.stop-during-backup=%s",
-		s.c.BackupStopContainerLabel,
-	)
 	containersToStop, err := s.cli.ContainerList(context.Background(), types.ContainerListOptions{
 		Filters: filters.NewArgs(filters.KeyValuePair{
 			Key:   "label",
-			Value: containerLabel,
+			Value: matchLabel,
 		}),
 	})
-
 	if err != nil {
 		return noop, fmt.Errorf("stopContainers: error querying for containers to stop: %w", err)
 	}
 
-	if len(containersToStop) == 0 {
+	allServices, err := s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
+	if err != nil {
+		return noop, fmt.Errorf("stopContainers: error querying for services: %w", err)
+	}
+	servicesToScaleDown, err := s.cli.ServiceList(context.Background(), types.ServiceListOptions{
+		Filters: filters.NewArgs(filters.KeyValuePair{
+			Key:   "label",
+			Value: matchLabel,
+		}),
+	})
+	if err != nil {
+		return noop, fmt.Errorf("stopContainers: error querying for services to scale down: %w", err)
+	}
+
+	if len(containersToStop) == 0 && len(servicesToScaleDown) == 0 {
 		return noop, nil
 	}
 
 	s.logger.Info(
 		fmt.Sprintf(
-			"Stopping %d container(s) labeled `%s` out of %d running container(s).",
+			"Stopping %d container(s) out of %d running container(s) and scaling down %d service(s) out of %d, as they were labeled %s.",
 			len(containersToStop),
-			containerLabel,
 			len(allContainers),
+			len(servicesToScaleDown),
+			len(allServices),
+			matchLabel,
 		),
 	)
 
@@ -385,12 +400,12 @@ func (s *script) stopContainers() (func() error, error) {
 	}
 
 	return func() error {
-		servicesRequiringUpdate := map[string]struct{}{}
+		servicesRequiringForceUpdate := map[string]struct{}{}
 
 		var restartErrors []error
 		for _, container := range stoppedContainers {
 			if swarmServiceName, ok := container.Labels["com.docker.swarm.service.name"]; ok {
-				servicesRequiringUpdate[swarmServiceName] = struct{}{}
+				servicesRequiringForceUpdate[swarmServiceName] = struct{}{}
 				continue
 			}
 			if err := s.cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{}); err != nil {
@@ -398,9 +413,9 @@ func (s *script) stopContainers() (func() error, error) {
 			}
 		}
 
-		if len(servicesRequiringUpdate) != 0 {
+		if len(servicesRequiringForceUpdate) != 0 {
 			services, _ := s.cli.ServiceList(context.Background(), types.ServiceListOptions{})
-			for serviceName := range servicesRequiringUpdate {
+			for serviceName := range servicesRequiringForceUpdate {
 				var serviceMatch swarm.Service
 				for _, service := range services {
 					if service.Spec.Name == serviceName {

test/services/docker-compose.yml

@@ -0,0 +1,69 @@
+# Copyright 2020-2021 - Offen Authors <hioffen@posteo.de>
+# SPDX-License-Identifier: Unlicense
+
+version: '3.8'
+
+services:
+  minio:
+    image: minio/minio:RELEASE.2020-08-04T23-10-51Z
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data'
+    volumes:
+      - backup_data:/data
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    depends_on:
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ
+      AWS_ENDPOINT: minio:9000
+      AWS_ENDPOINT_PROTO: http
+      AWS_S3_BUCKET_NAME: backup
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: 7
+      BACKUP_PRUNING_LEEWAY: 5s
+    volumes:
+      - pg_data:/backup/pg_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  offen:
+    image: offen/offen:latest
+    healthcheck:
+      disable: true
+    deploy:
+      labels:
+        - docker-volume-backup.stop-during-backup=true
+      replicas: 2
+      restart_policy:
+        condition: on-failure
+
+  pg:
+    image: postgres:14-alpine
+    environment:
+      POSTGRES_PASSWORD: example
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    deploy:
+      restart_policy:
+        condition: on-failure
+
+volumes:
+  backup_data:
+    name: backup_data
+  pg_data:
+    name: pg_data

test/services/run.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker swarm init
+
+docker stack deploy --compose-file=docker-compose.yml test_stack
+
+while [ -z $(docker ps -q -f name=backup) ]; do
+  info "Backup container not ready yet. Retrying."
+  sleep 1
+done
+
+sleep 20
+
+docker exec $(docker ps -q -f name=backup) backup
+
+docker run --rm \
+  -v backup_data:/data alpine \
+  ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
+
+pass "Found relevant files in untared backup."
+
+sleep 5
+expect_running_containers "5"