add aws secret handling (#161)

* add aws secret handling

* make it look go-ish

* fix tests

* whitespace

* sleep a bit
This commit is contained in:
pixxon 2022-10-12 19:14:57 +02:00 committed by GitHub
parent 00c83dfac7
commit b5cc1262e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 202 additions and 2 deletions

View File

@ -196,6 +196,14 @@ You can populate below template according to your requirements and use it as you
# AWS_ACCESS_KEY_ID="<xxx>" # AWS_ACCESS_KEY_ID="<xxx>"
# AWS_SECRET_ACCESS_KEY="<xxx>" # AWS_SECRET_ACCESS_KEY="<xxx>"
# It is possible to provide the keys in files, allowing to hide the sensitive data.
# These values have a higher priority than the ones above, meaning if both are set
# the values from the files will be used.
# This option is most useful with Docker [secrets](https://docs.docker.com/engine/swarm/secrets/).
# AWS_ACCESS_KEY_ID_FILE="/path/to/file"
# AWS_SECRET_ACCESS_KEY_FILE="/path/to/file"
# Instead of providing static credentials, you can also use IAM instance profiles # Instead of providing static credentials, you can also use IAM instance profiles
# or similar to provide authentication. Some possible configuration options on AWS: # or similar to provide authentication. Some possible configuration options on AWS:
# - EC2: http://169.254.169.254 # - EC2: http://169.254.169.254
@ -947,6 +955,38 @@ volumes:
data: data:
``` ```
### Backing up to MinIO (using Docker secrets)
```yml
version: '3'
services:
# ... define other services using the `data` volume here
backup:
image: offen/docker-volume-backup:v2
environment:
AWS_ENDPOINT: minio.example.com
AWS_S3_BUCKET_NAME: backup-bucket
AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_access_key
AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_secret_key
volumes:
- data:/backup/my-app-backup:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
secrets:
- minio_access_key
- minio_secret_key
volumes:
data:
secrets:
minio_access_key:
# ... define how secret is accessed
minio_secret_key:
# ... define how secret is accessed
```
### Backing up to WebDAV ### Backing up to WebDAV
```yml ```yml

View File

@ -4,6 +4,7 @@
package main package main
import ( import (
"os"
"fmt" "fmt"
"regexp" "regexp"
"time" "time"
@ -19,7 +20,9 @@ type Config struct {
AwsEndpointInsecure bool `split_words:"true"` AwsEndpointInsecure bool `split_words:"true"`
AwsStorageClass string `split_words:"true"` AwsStorageClass string `split_words:"true"`
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
AwsSecretAccessKey string `split_words:"true"` AwsSecretAccessKey string `split_words:"true"`
AwsSecretAccessKeyFile string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"` AwsIamRoleEndpoint string `split_words:"true"`
BackupSources string `split_words:"true" default:"/backup"` BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
@ -58,6 +61,17 @@ type Config struct {
LockTimeout time.Duration `split_words:"true" default:"60m"` LockTimeout time.Duration `split_words:"true" default:"60m"`
} }
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
if secretPath != "" {
data, err := os.ReadFile(secretPath)
if err != nil {
return "", fmt.Errorf("resolveSecret: error reading secret path: %w", err)
}
return string(data), nil
}
return envVar, nil
}
type RegexpDecoder struct { type RegexpDecoder struct {
Re *regexp.Regexp Re *regexp.Regexp
} }

View File

@ -121,10 +121,18 @@ func newScript() (*script, error) {
} }
if s.c.AwsS3BucketName != "" { if s.c.AwsS3BucketName != "" {
accessKeyID, err := s.c.resolveSecret(s.c.AwsAccessKeyID, s.c.AwsAccessKeyIDFile)
if err != nil {
return nil, fmt.Errorf("newScript: error resolving AwsAccessKeyID: %w", err)
}
secretAccessKey, err := s.c.resolveSecret(s.c.AwsSecretAccessKey, s.c.AwsSecretAccessKeyFile)
if err != nil {
return nil, fmt.Errorf("newScript: error resolving AwsSecretAccessKey: %w", err)
}
s3Config := s3.Config{ s3Config := s3.Config{
Endpoint: s.c.AwsEndpoint, Endpoint: s.c.AwsEndpoint,
AccessKeyID: s.c.AwsAccessKeyID, AccessKeyID: accessKeyID,
SecretAccessKey: s.c.AwsSecretAccessKey, SecretAccessKey: secretAccessKey,
IamRoleEndpoint: s.c.AwsIamRoleEndpoint, IamRoleEndpoint: s.c.AwsIamRoleEndpoint,
EndpointProto: s.c.AwsEndpointProto, EndpointProto: s.c.AwsEndpointProto,
EndpointInsecure: s.c.AwsEndpointInsecure, EndpointInsecure: s.c.AwsEndpointInsecure,

View File

@ -0,0 +1,89 @@
# Copyright 2020-2021 - Offen Authors <hioffen@posteo.de>
# SPDX-License-Identifier: Unlicense
version: '3.8'
services:
minio_setup:
image: alpine:latest
deploy:
restart_policy:
condition: none
volumes:
- backup_data:/data
command: mkdir -p /data/backup
minio:
image: minio/minio:RELEASE.2021-12-20T22-07-16Z
deploy:
restart_policy:
condition: on-failure
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
command: minio server /data
volumes:
- backup_data:/data
secrets:
- minio_root_user
- minio_root_password
depends_on:
- minio_setup
backup:
image: offen/docker-volume-backup:${TEST_VERSION:-canary}
depends_on:
- minio
deploy:
restart_policy:
condition: on-failure
environment:
AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_root_user
AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_root_password
AWS_ENDPOINT: minio:9000
AWS_ENDPOINT_PROTO: http
AWS_S3_BUCKET_NAME: backup
BACKUP_FILENAME: test.tar.gz
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
BACKUP_RETENTION_DAYS: 7
BACKUP_PRUNING_LEEWAY: 5s
volumes:
- pg_data:/backup/pg_data:ro
- /var/run/docker.sock:/var/run/docker.sock
secrets:
- minio_root_user
- minio_root_password
offen:
image: offen/offen:latest
labels:
- docker-volume-backup.stop-during-backup=true
healthcheck:
disable: true
deploy:
replicas: 2
restart_policy:
condition: on-failure
pg:
image: postgres:14-alpine
environment:
POSTGRES_PASSWORD: example
labels:
- docker-volume-backup.stop-during-backup=true
volumes:
- pg_data:/var/lib/postgresql/data
deploy:
restart_policy:
condition: on-failure
volumes:
backup_data:
name: backup_data
pg_data:
secrets:
minio_root_user:
external: true
minio_root_password:
external: true

44
test/secret/run.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/sh
set -e
cd $(dirname $0)
. ../util.sh
current_test=$(basename $(pwd))
docker swarm init
printf "test" | docker secret create minio_root_user -
printf "GMusLtUmILge2by+z890kQ" | docker secret create minio_root_password -
docker stack deploy --compose-file=docker-compose.yml test_stack
while [ -z $(docker ps -q -f name=backup) ]; do
info "Backup container not ready yet. Retrying."
sleep 1
done
sleep 20
docker exec $(docker ps -q -f name=backup) backup
docker run --rm -it \
-v backup_data:/data alpine \
ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
pass "Found relevant files in untared backup."
sleep 5
expect_running_containers "5"
docker stack rm test_stack
docker secret rm minio_root_password
docker secret rm minio_root_user
docker swarm leave --force
sleep 10
docker volume rm backup_data
docker volume rm test_stack_pg_data

View File

@ -30,3 +30,8 @@ expect_running_containers "5"
docker stack rm test_stack docker stack rm test_stack
docker swarm leave --force docker swarm leave --force
sleep 10
docker volume rm backup_data
docker volume rm test_stack_pg_data