add aws secret handling (#161)

* add aws secret handling

* make it look go-ish

* fix tests

* whitespace

* sleep a bit

README.md

@@ -196,6 +196,14 @@ You can populate below template according to your requirements and use it as you
 # AWS_ACCESS_KEY_ID="<xxx>"
 # AWS_SECRET_ACCESS_KEY="<xxx>"
 
+# It is possible to provide the keys in files, allowing to hide the sensitive data.
+# These values have a higher priority than the ones above, meaning if both are set
+# the values from the files will be used.
+# This option is most useful with Docker [secrets](https://docs.docker.com/engine/swarm/secrets/).
+
+# AWS_ACCESS_KEY_ID_FILE="/path/to/file"
+# AWS_SECRET_ACCESS_KEY_FILE="/path/to/file"
+
 # Instead of providing static credentials, you can also use IAM instance profiles
 # or similar to provide authentication. Some possible configuration options on AWS:
 # - EC2: http://169.254.169.254
@@ -947,6 +955,38 @@ volumes:
   data:
 ```
 
+
+### Backing up to MinIO (using Docker secrets)
+
+```yml
+version: '3'
+
+services:
+  # ... define other services using the `data` volume here
+  backup:
+    image: offen/docker-volume-backup:v2
+    environment:
+      AWS_ENDPOINT: minio.example.com
+      AWS_S3_BUCKET_NAME: backup-bucket
+      AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_access_key
+      AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_secret_key
+    volumes:
+      - data:/backup/my-app-backup:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    secrets:
+      - minio_access_key
+      - minio_secret_key
+
+volumes:
+  data:
+
+secrets:
+  minio_access_key:
+    # ... define how secret is accessed
+  minio_secret_key:
+    # ... define how secret is accessed
+```
+
 ### Backing up to WebDAV
 
 ```yml

cmd/backup/config.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"os"
 	"fmt"
 	"regexp"
 	"time"
@@ -19,7 +20,9 @@ type Config struct {
 	AwsEndpointInsecure        bool          `split_words:"true"`
 	AwsStorageClass            string        `split_words:"true"`
 	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
+	AwsAccessKeyIDFile         string        `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
 	AwsSecretAccessKey         string        `split_words:"true"`
+	AwsSecretAccessKeyFile     string        `split_words:"true"`
 	AwsIamRoleEndpoint         string        `split_words:"true"`
 	BackupSources              string        `split_words:"true" default:"/backup"`
 	BackupFilename             string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
@@ -58,6 +61,17 @@ type Config struct {
 	LockTimeout                time.Duration `split_words:"true" default:"60m"`
 }
 
+func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
+	if secretPath != "" {
+		data, err := os.ReadFile(secretPath)
+		if err != nil {
+			return "", fmt.Errorf("resolveSecret: error reading secret path: %w", err)
+		}
+		return string(data), nil
+	}
+	return envVar, nil
+}
+
 type RegexpDecoder struct {
 	Re *regexp.Regexp
 }

cmd/backup/script.go

@@ -121,10 +121,18 @@ func newScript() (*script, error) {
 	}
 
 	if s.c.AwsS3BucketName != "" {
+		accessKeyID, err := s.c.resolveSecret(s.c.AwsAccessKeyID, s.c.AwsAccessKeyIDFile)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: error resolving AwsAccessKeyID: %w", err)
+		}
+		secretAccessKey, err := s.c.resolveSecret(s.c.AwsSecretAccessKey, s.c.AwsSecretAccessKeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("newScript: error resolving AwsSecretAccessKey: %w", err)
+		}
 		s3Config := s3.Config{
 			Endpoint:         s.c.AwsEndpoint,
-			AccessKeyID:      s.c.AwsAccessKeyID,
-			SecretAccessKey:  s.c.AwsSecretAccessKey,
+			AccessKeyID:      accessKeyID,
+			SecretAccessKey:  secretAccessKey,
 			IamRoleEndpoint:  s.c.AwsIamRoleEndpoint,
 			EndpointProto:    s.c.AwsEndpointProto,
 			EndpointInsecure: s.c.AwsEndpointInsecure,

test/secret/docker-compose.yml

@@ -0,0 +1,89 @@
+# Copyright 2020-2021 - Offen Authors <hioffen@posteo.de>
+# SPDX-License-Identifier: Unlicense
+
+version: '3.8'
+
+services:
+  minio_setup:
+    image: alpine:latest
+    deploy:
+      restart_policy:
+        condition: none
+    volumes:
+      - backup_data:/data
+    command: mkdir -p /data/backup
+    
+  minio:
+    image: minio/minio:RELEASE.2021-12-20T22-07-16Z
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
+      MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
+    command: minio server /data
+    volumes:
+      - backup_data:/data
+    secrets:
+      - minio_root_user
+      - minio_root_password
+    depends_on:
+      - minio_setup
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    depends_on:
+      - minio
+    deploy:
+      restart_policy:
+        condition: on-failure
+    environment:
+      AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_root_user
+      AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_root_password
+      AWS_ENDPOINT: minio:9000
+      AWS_ENDPOINT_PROTO: http
+      AWS_S3_BUCKET_NAME: backup
+      BACKUP_FILENAME: test.tar.gz
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: 7
+      BACKUP_PRUNING_LEEWAY: 5s
+    volumes:
+      - pg_data:/backup/pg_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+    secrets:
+      - minio_root_user
+      - minio_root_password
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    healthcheck:
+      disable: true
+    deploy:
+      replicas: 2
+      restart_policy:
+        condition: on-failure
+
+  pg:
+    image: postgres:14-alpine
+    environment:
+      POSTGRES_PASSWORD: example
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - pg_data:/var/lib/postgresql/data
+    deploy:
+      restart_policy:
+        condition: on-failure
+
+volumes:
+  backup_data:
+    name: backup_data
+  pg_data:
+
+secrets:
+  minio_root_user:
+    external: true
+  minio_root_password:
+    external: true

test/secret/run.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -e
+
+cd $(dirname $0)
+. ../util.sh
+current_test=$(basename $(pwd))
+
+docker swarm init
+
+printf "test" | docker secret create minio_root_user -
+printf "GMusLtUmILge2by+z890kQ" | docker secret create minio_root_password -
+
+docker stack deploy --compose-file=docker-compose.yml test_stack
+
+while [ -z $(docker ps -q -f name=backup) ]; do
+  info "Backup container not ready yet. Retrying."
+  sleep 1
+done
+
+sleep 20
+
+docker exec $(docker ps -q -f name=backup) backup
+
+docker run --rm -it \
+  -v backup_data:/data alpine \
+  ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
+
+pass "Found relevant files in untared backup."
+
+sleep 5
+expect_running_containers "5"
+
+docker stack rm test_stack
+
+docker secret rm minio_root_password
+docker secret rm minio_root_user
+
+docker swarm leave --force
+
+sleep 10
+
+docker volume rm backup_data
+docker volume rm test_stack_pg_data

test/swarm/run.sh

@@ -30,3 +30,8 @@ expect_running_containers "5"
 
 docker stack rm test_stack
 docker swarm leave --force
+
+sleep 10
+
+docker volume rm backup_data
+docker volume rm test_stack_pg_data