Support _FILE based configuration values (#264)

* Replace envconfig with env

* Adjust config options and processing

* Added _FILE variant for all password vars.

* Try pathenvconfig

* Revert everything so far

* Use our fork of envconfig with custom lookup

* Use our fork of envconfig with custom lookup

* Test compose timeout option

* Remove secret resolving and specific _FILE config

* Fix timing issue in swarm tests

* Revert "Test compose timeout option"

This reverts commit ab50b21748, reversing
changes made to 0282514b2b.

Revert "Test compose timeout option"

This reverts commit 0282514b2b.

* Use offen/envconfig v1.5.0

* Add info about _FILE in README

* Value > File. Panic on file error. Panic on duplicate presence.

* Test panic on duplicate vars and panic on file error.
This commit is contained in:
MaxJa4 2023-08-30 20:14:24 +02:00 committed by GitHub
parent 24a6ec9480
commit 43c4961116
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 42 additions and 35 deletions

View File

@ -139,6 +139,9 @@ Documentation references Docker Hub, but all examples will work using ghcr.io ju
## Configuration reference ## Configuration reference
Backup targets, schedule and retention are configured in environment variables. Backup targets, schedule and retention are configured in environment variables.
Note: You can use any environment variable from below also with a `_FILE` suffix to be able to load the value from a file. This is usually useful when using [Docker Secrets](https://docs.docker.com/engine/swarm/secrets/) or similar.
You can populate below template according to your requirements and use it as your `env_file`: You can populate below template according to your requirements and use it as your `env_file`:
```ini ```ini
@ -233,14 +236,6 @@ You can populate below template according to your requirements and use it as you
# AWS_ACCESS_KEY_ID="<xxx>" # AWS_ACCESS_KEY_ID="<xxx>"
# AWS_SECRET_ACCESS_KEY="<xxx>" # AWS_SECRET_ACCESS_KEY="<xxx>"
# It is possible to provide the keys in files, allowing to hide the sensitive data.
# These values have a higher priority than the ones above, meaning if both are set
# the values from the files will be used.
# This option is most useful with Docker [secrets](https://docs.docker.com/engine/swarm/secrets/).
# AWS_ACCESS_KEY_ID_FILE="/path/to/file"
# AWS_SECRET_ACCESS_KEY_FILE="/path/to/file"
# Instead of providing static credentials, you can also use IAM instance profiles # Instead of providing static credentials, you can also use IAM instance profiles
# or similar to provide authentication. Some possible configuration options on AWS: # or similar to provide authentication. Some possible configuration options on AWS:
# - EC2: http://169.254.169.254 # - EC2: http://169.254.169.254

View File

@ -24,9 +24,7 @@ type Config struct {
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
AwsStorageClass string `split_words:"true"` AwsStorageClass string `split_words:"true"`
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
AwsSecretAccessKey string `split_words:"true"` AwsSecretAccessKey string `split_words:"true"`
AwsSecretAccessKeyFile string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"` AwsIamRoleEndpoint string `split_words:"true"`
AwsPartSize int64 `split_words:"true"` AwsPartSize int64 `split_words:"true"`
BackupCompression CompressionType `split_words:"true" default:"gz"` BackupCompression CompressionType `split_words:"true" default:"gz"`
@ -80,17 +78,6 @@ type Config struct {
DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"` DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"`
} }
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
if secretPath == "" {
return envVar, nil
}
data, err := os.ReadFile(secretPath)
if err != nil {
return "", fmt.Errorf("resolveSecret: error reading secret path: %w", err)
}
return string(data), nil
}
type CompressionType string type CompressionType string
func (c *CompressionType) Decode(v string) error { func (c *CompressionType) Decode(v string) error {

View File

@ -35,8 +35,8 @@ import (
"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm" "github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client" "github.com/docker/docker/client"
"github.com/kelseyhightower/envconfig"
"github.com/leekchan/timeutil" "github.com/leekchan/timeutil"
"github.com/offen/envconfig"
"github.com/otiai10/copy" "github.com/otiai10/copy"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -89,6 +89,28 @@ func newScript() (*script, error) {
return nil return nil
}) })
envconfig.Lookup = func(key string) (string, bool) {
value, okValue := os.LookupEnv(key)
location, okFile := os.LookupEnv(key + "_FILE")
switch {
case okValue && !okFile: // only value
return value, true
case !okValue && okFile: // only file
contents, err := os.ReadFile(location)
if err != nil {
s.must(fmt.Errorf("newScript: failed to read %s! Error: %s", location, err))
return "", false
}
return string(contents), true
case okValue && okFile: // both
s.must(fmt.Errorf("newScript: both %s and %s are set!", key, key+"_FILE"))
return "", false
default: // neither, ignore
return "", false
}
}
if err := envconfig.Process("", s.c); err != nil { if err := envconfig.Process("", s.c); err != nil {
return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err) return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
} }
@ -137,18 +159,10 @@ func newScript() (*script, error) {
} }
if s.c.AwsS3BucketName != "" { if s.c.AwsS3BucketName != "" {
accessKeyID, err := s.c.resolveSecret(s.c.AwsAccessKeyID, s.c.AwsAccessKeyIDFile)
if err != nil {
return nil, fmt.Errorf("newScript: error resolving AwsAccessKeyID: %w", err)
}
secretAccessKey, err := s.c.resolveSecret(s.c.AwsSecretAccessKey, s.c.AwsSecretAccessKeyFile)
if err != nil {
return nil, fmt.Errorf("newScript: error resolving AwsSecretAccessKey: %w", err)
}
s3Config := s3.Config{ s3Config := s3.Config{
Endpoint: s.c.AwsEndpoint, Endpoint: s.c.AwsEndpoint,
AccessKeyID: accessKeyID, AccessKeyID: s.c.AwsAccessKeyID,
SecretAccessKey: secretAccessKey, SecretAccessKey: s.c.AwsSecretAccessKey,
IamRoleEndpoint: s.c.AwsIamRoleEndpoint, IamRoleEndpoint: s.c.AwsIamRoleEndpoint,
EndpointProto: s.c.AwsEndpointProto, EndpointProto: s.c.AwsEndpointProto,
EndpointInsecure: s.c.AwsEndpointInsecure, EndpointInsecure: s.c.AwsEndpointInsecure,

2
go.mod
View File

@ -9,10 +9,10 @@ require (
github.com/cosiner/argv v0.1.0 github.com/cosiner/argv v0.1.0
github.com/docker/docker v24.0.5+incompatible github.com/docker/docker v24.0.5+incompatible
github.com/gofrs/flock v0.8.1 github.com/gofrs/flock v0.8.1
github.com/kelseyhightower/envconfig v1.4.0
github.com/klauspost/compress v1.16.7 github.com/klauspost/compress v1.16.7
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
github.com/minio/minio-go/v7 v7.0.62 github.com/minio/minio-go/v7 v7.0.62
github.com/offen/envconfig v1.5.0
github.com/otiai10/copy v1.11.0 github.com/otiai10/copy v1.11.0
github.com/pkg/sftp v1.13.6 github.com/pkg/sftp v1.13.6
github.com/studio-b12/gowebdav v0.9.0 github.com/studio-b12/gowebdav v0.9.0

4
go.sum
View File

@ -451,8 +451,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
@ -527,6 +525,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/offen/envconfig v1.5.0 h1:LHL4wYIDVeoGxSDI40MShmWfss3gYUlCdstfSiSq4Fk=
github.com/offen/envconfig v1.5.0/go.mod h1:L7ny7R+4JWH3VVnZ+ARHvZysWUiZ2eQcm3L0imU9ACY=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=

View File

@ -31,6 +31,16 @@ pass "Found relevant files in untared backup."
sleep 5 sleep 5
expect_running_containers "5" expect_running_containers "5"
docker exec -e AWS_ACCESS_KEY_ID=test $(docker ps -q -f name=backup) backup \
&& fail "Backup should have failed due to duplicate env variables."
pass "Backup failed due to duplicate env variables."
docker exec -e AWS_ACCESS_KEY_ID_FILE=/tmp/nonexistant $(docker ps -q -f name=backup) backup \
&& fail "Backup should have failed due to non existing file env variable."
pass "Backup failed due to non existing file env variable."
docker stack rm test_stack docker stack rm test_stack
docker secret rm minio_root_password docker secret rm minio_root_password

View File

@ -29,6 +29,7 @@ sleep 5
expect_running_containers "5" expect_running_containers "5"
docker stack rm test_stack docker stack rm test_stack
sleep 1
docker swarm leave --force docker swarm leave --force
sleep 10 sleep 10