mirror of
https://github.com/offen/docker-volume-backup.git
synced 2024-11-10 00:30:29 +01:00
add aws secret handling (#161)
* add aws secret handling * make it look go-ish * fix tests * whitespace * sleep a bit
This commit is contained in:
parent
00c83dfac7
commit
b5cc1262e2
40
README.md
40
README.md
@ -196,6 +196,14 @@ You can populate below template according to your requirements and use it as you
|
|||||||
# AWS_ACCESS_KEY_ID="<xxx>"
|
# AWS_ACCESS_KEY_ID="<xxx>"
|
||||||
# AWS_SECRET_ACCESS_KEY="<xxx>"
|
# AWS_SECRET_ACCESS_KEY="<xxx>"
|
||||||
|
|
||||||
|
# It is possible to provide the keys in files, allowing to hide the sensitive data.
|
||||||
|
# These values have a higher priority than the ones above, meaning if both are set
|
||||||
|
# the values from the files will be used.
|
||||||
|
# This option is most useful with Docker [secrets](https://docs.docker.com/engine/swarm/secrets/).
|
||||||
|
|
||||||
|
# AWS_ACCESS_KEY_ID_FILE="/path/to/file"
|
||||||
|
# AWS_SECRET_ACCESS_KEY_FILE="/path/to/file"
|
||||||
|
|
||||||
# Instead of providing static credentials, you can also use IAM instance profiles
|
# Instead of providing static credentials, you can also use IAM instance profiles
|
||||||
# or similar to provide authentication. Some possible configuration options on AWS:
|
# or similar to provide authentication. Some possible configuration options on AWS:
|
||||||
# - EC2: http://169.254.169.254
|
# - EC2: http://169.254.169.254
|
||||||
@ -947,6 +955,38 @@ volumes:
|
|||||||
data:
|
data:
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### Backing up to MinIO (using Docker secrets)
|
||||||
|
|
||||||
|
```yml
|
||||||
|
version: '3'
|
||||||
|
|
||||||
|
services:
|
||||||
|
# ... define other services using the `data` volume here
|
||||||
|
backup:
|
||||||
|
image: offen/docker-volume-backup:v2
|
||||||
|
environment:
|
||||||
|
AWS_ENDPOINT: minio.example.com
|
||||||
|
AWS_S3_BUCKET_NAME: backup-bucket
|
||||||
|
AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_access_key
|
||||||
|
AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_secret_key
|
||||||
|
volumes:
|
||||||
|
- data:/backup/my-app-backup:ro
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
secrets:
|
||||||
|
- minio_access_key
|
||||||
|
- minio_secret_key
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
data:
|
||||||
|
|
||||||
|
secrets:
|
||||||
|
minio_access_key:
|
||||||
|
# ... define how secret is accessed
|
||||||
|
minio_secret_key:
|
||||||
|
# ... define how secret is accessed
|
||||||
|
```
|
||||||
|
|
||||||
### Backing up to WebDAV
|
### Backing up to WebDAV
|
||||||
|
|
||||||
```yml
|
```yml
|
||||||
|
@ -4,6 +4,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
"time"
|
"time"
|
||||||
@ -19,7 +20,9 @@ type Config struct {
|
|||||||
AwsEndpointInsecure bool `split_words:"true"`
|
AwsEndpointInsecure bool `split_words:"true"`
|
||||||
AwsStorageClass string `split_words:"true"`
|
AwsStorageClass string `split_words:"true"`
|
||||||
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
|
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
|
||||||
|
AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
|
||||||
AwsSecretAccessKey string `split_words:"true"`
|
AwsSecretAccessKey string `split_words:"true"`
|
||||||
|
AwsSecretAccessKeyFile string `split_words:"true"`
|
||||||
AwsIamRoleEndpoint string `split_words:"true"`
|
AwsIamRoleEndpoint string `split_words:"true"`
|
||||||
BackupSources string `split_words:"true" default:"/backup"`
|
BackupSources string `split_words:"true" default:"/backup"`
|
||||||
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
|
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
|
||||||
@ -58,6 +61,17 @@ type Config struct {
|
|||||||
LockTimeout time.Duration `split_words:"true" default:"60m"`
|
LockTimeout time.Duration `split_words:"true" default:"60m"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
|
||||||
|
if secretPath != "" {
|
||||||
|
data, err := os.ReadFile(secretPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("resolveSecret: error reading secret path: %w", err)
|
||||||
|
}
|
||||||
|
return string(data), nil
|
||||||
|
}
|
||||||
|
return envVar, nil
|
||||||
|
}
|
||||||
|
|
||||||
type RegexpDecoder struct {
|
type RegexpDecoder struct {
|
||||||
Re *regexp.Regexp
|
Re *regexp.Regexp
|
||||||
}
|
}
|
||||||
|
@ -121,10 +121,18 @@ func newScript() (*script, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if s.c.AwsS3BucketName != "" {
|
if s.c.AwsS3BucketName != "" {
|
||||||
|
accessKeyID, err := s.c.resolveSecret(s.c.AwsAccessKeyID, s.c.AwsAccessKeyIDFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("newScript: error resolving AwsAccessKeyID: %w", err)
|
||||||
|
}
|
||||||
|
secretAccessKey, err := s.c.resolveSecret(s.c.AwsSecretAccessKey, s.c.AwsSecretAccessKeyFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("newScript: error resolving AwsSecretAccessKey: %w", err)
|
||||||
|
}
|
||||||
s3Config := s3.Config{
|
s3Config := s3.Config{
|
||||||
Endpoint: s.c.AwsEndpoint,
|
Endpoint: s.c.AwsEndpoint,
|
||||||
AccessKeyID: s.c.AwsAccessKeyID,
|
AccessKeyID: accessKeyID,
|
||||||
SecretAccessKey: s.c.AwsSecretAccessKey,
|
SecretAccessKey: secretAccessKey,
|
||||||
IamRoleEndpoint: s.c.AwsIamRoleEndpoint,
|
IamRoleEndpoint: s.c.AwsIamRoleEndpoint,
|
||||||
EndpointProto: s.c.AwsEndpointProto,
|
EndpointProto: s.c.AwsEndpointProto,
|
||||||
EndpointInsecure: s.c.AwsEndpointInsecure,
|
EndpointInsecure: s.c.AwsEndpointInsecure,
|
||||||
|
89
test/secret/docker-compose.yml
Normal file
89
test/secret/docker-compose.yml
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
# Copyright 2020-2021 - Offen Authors <hioffen@posteo.de>
|
||||||
|
# SPDX-License-Identifier: Unlicense
|
||||||
|
|
||||||
|
version: '3.8'
|
||||||
|
|
||||||
|
services:
|
||||||
|
minio_setup:
|
||||||
|
image: alpine:latest
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: none
|
||||||
|
volumes:
|
||||||
|
- backup_data:/data
|
||||||
|
command: mkdir -p /data/backup
|
||||||
|
|
||||||
|
minio:
|
||||||
|
image: minio/minio:RELEASE.2021-12-20T22-07-16Z
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
environment:
|
||||||
|
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||||
|
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||||
|
command: minio server /data
|
||||||
|
volumes:
|
||||||
|
- backup_data:/data
|
||||||
|
secrets:
|
||||||
|
- minio_root_user
|
||||||
|
- minio_root_password
|
||||||
|
depends_on:
|
||||||
|
- minio_setup
|
||||||
|
|
||||||
|
backup:
|
||||||
|
image: offen/docker-volume-backup:${TEST_VERSION:-canary}
|
||||||
|
depends_on:
|
||||||
|
- minio
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_root_user
|
||||||
|
AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_root_password
|
||||||
|
AWS_ENDPOINT: minio:9000
|
||||||
|
AWS_ENDPOINT_PROTO: http
|
||||||
|
AWS_S3_BUCKET_NAME: backup
|
||||||
|
BACKUP_FILENAME: test.tar.gz
|
||||||
|
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
|
||||||
|
BACKUP_RETENTION_DAYS: 7
|
||||||
|
BACKUP_PRUNING_LEEWAY: 5s
|
||||||
|
volumes:
|
||||||
|
- pg_data:/backup/pg_data:ro
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
secrets:
|
||||||
|
- minio_root_user
|
||||||
|
- minio_root_password
|
||||||
|
|
||||||
|
offen:
|
||||||
|
image: offen/offen:latest
|
||||||
|
labels:
|
||||||
|
- docker-volume-backup.stop-during-backup=true
|
||||||
|
healthcheck:
|
||||||
|
disable: true
|
||||||
|
deploy:
|
||||||
|
replicas: 2
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
|
||||||
|
pg:
|
||||||
|
image: postgres:14-alpine
|
||||||
|
environment:
|
||||||
|
POSTGRES_PASSWORD: example
|
||||||
|
labels:
|
||||||
|
- docker-volume-backup.stop-during-backup=true
|
||||||
|
volumes:
|
||||||
|
- pg_data:/var/lib/postgresql/data
|
||||||
|
deploy:
|
||||||
|
restart_policy:
|
||||||
|
condition: on-failure
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
backup_data:
|
||||||
|
name: backup_data
|
||||||
|
pg_data:
|
||||||
|
|
||||||
|
secrets:
|
||||||
|
minio_root_user:
|
||||||
|
external: true
|
||||||
|
minio_root_password:
|
||||||
|
external: true
|
44
test/secret/run.sh
Executable file
44
test/secret/run.sh
Executable file
@ -0,0 +1,44 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
cd $(dirname $0)
|
||||||
|
. ../util.sh
|
||||||
|
current_test=$(basename $(pwd))
|
||||||
|
|
||||||
|
docker swarm init
|
||||||
|
|
||||||
|
printf "test" | docker secret create minio_root_user -
|
||||||
|
printf "GMusLtUmILge2by+z890kQ" | docker secret create minio_root_password -
|
||||||
|
|
||||||
|
docker stack deploy --compose-file=docker-compose.yml test_stack
|
||||||
|
|
||||||
|
while [ -z $(docker ps -q -f name=backup) ]; do
|
||||||
|
info "Backup container not ready yet. Retrying."
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
sleep 20
|
||||||
|
|
||||||
|
docker exec $(docker ps -q -f name=backup) backup
|
||||||
|
|
||||||
|
docker run --rm -it \
|
||||||
|
-v backup_data:/data alpine \
|
||||||
|
ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION'
|
||||||
|
|
||||||
|
pass "Found relevant files in untared backup."
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
expect_running_containers "5"
|
||||||
|
|
||||||
|
docker stack rm test_stack
|
||||||
|
|
||||||
|
docker secret rm minio_root_password
|
||||||
|
docker secret rm minio_root_user
|
||||||
|
|
||||||
|
docker swarm leave --force
|
||||||
|
|
||||||
|
sleep 10
|
||||||
|
|
||||||
|
docker volume rm backup_data
|
||||||
|
docker volume rm test_stack_pg_data
|
@ -30,3 +30,8 @@ expect_running_containers "5"
|
|||||||
|
|
||||||
docker stack rm test_stack
|
docker stack rm test_stack
|
||||||
docker swarm leave --force
|
docker swarm leave --force
|
||||||
|
|
||||||
|
sleep 10
|
||||||
|
|
||||||
|
docker volume rm backup_data
|
||||||
|
docker volume rm test_stack_pg_data
|
||||||
|
Loading…
Reference in New Issue
Block a user