From 43c49611166408ef3b52fa979fc07a318c1ece6f Mon Sep 17 00:00:00 2001 From: MaxJa4 <74194322+MaxJa4@users.noreply.github.com> Date: Wed, 30 Aug 2023 20:14:24 +0200 Subject: [PATCH] Support _FILE based configuration values (#264) * Replace envconfig with env * Adjust config options and processing * Added _FILE variant for all password vars. * Try pathenvconfig * Revert everything so far * Use our fork of envconfig with custom lookup * Use our fork of envconfig with custom lookup * Test compose timeout option * Remove secret resolving and specific _FILE config * Fix timing issue in swarm tests * Revert "Test compose timeout option" This reverts commit ab50b217486372f6568fdfef4bd7134796833343, reversing changes made to 0282514b2b735c913b31700e35bfa9534fd93375. Revert "Test compose timeout option" This reverts commit 0282514b2b735c913b31700e35bfa9534fd93375. * Use offen/envconfig v1.5.0 * Add info about _FILE in README * Value > File. Panic on file error. Panic on duplicate presence. * Test panic on duplicate vars and panic on file error. --- README.md | 11 +++-------- cmd/backup/config.go | 13 ------------- cmd/backup/script.go | 36 +++++++++++++++++++++++++----------- go.mod | 2 +- go.sum | 4 ++-- test/secrets/run.sh | 10 ++++++++++ test/swarm/run.sh | 1 + 7 files changed, 42 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 418393f..2c9e1b3 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,9 @@ Documentation references Docker Hub, but all examples will work using ghcr.io ju ## Configuration reference Backup targets, schedule and retention are configured in environment variables. + +Note: You can use any environment variable from below also with a `_FILE` suffix to be able to load the value from a file. This is usually useful when using [Docker Secrets](https://docs.docker.com/engine/swarm/secrets/) or similar. + You can populate below template according to your requirements and use it as your `env_file`: ```ini @@ -233,14 +236,6 @@ You can populate below template according to your requirements and use it as you # AWS_ACCESS_KEY_ID="" # AWS_SECRET_ACCESS_KEY="" -# It is possible to provide the keys in files, allowing to hide the sensitive data. -# These values have a higher priority than the ones above, meaning if both are set -# the values from the files will be used. -# This option is most useful with Docker [secrets](https://docs.docker.com/engine/swarm/secrets/). - -# AWS_ACCESS_KEY_ID_FILE="/path/to/file" -# AWS_SECRET_ACCESS_KEY_FILE="/path/to/file" - # Instead of providing static credentials, you can also use IAM instance profiles # or similar to provide authentication. Some possible configuration options on AWS: # - EC2: http://169.254.169.254 diff --git a/cmd/backup/config.go b/cmd/backup/config.go index d6805f9..82f4b0a 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -24,9 +24,7 @@ type Config struct { AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` AwsStorageClass string `split_words:"true"` AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` - AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"` AwsSecretAccessKey string `split_words:"true"` - AwsSecretAccessKeyFile string `split_words:"true"` AwsIamRoleEndpoint string `split_words:"true"` AwsPartSize int64 `split_words:"true"` BackupCompression CompressionType `split_words:"true" default:"gz"` @@ -80,17 +78,6 @@ type Config struct { DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"` } -func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { - if secretPath == "" { - return envVar, nil - } - data, err := os.ReadFile(secretPath) - if err != nil { - return "", fmt.Errorf("resolveSecret: error reading secret path: %w", err) - } - return string(data), nil -} - type CompressionType string func (c *CompressionType) Decode(v string) error { diff --git a/cmd/backup/script.go b/cmd/backup/script.go index 5a455c3..0c12ad7 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -35,8 +35,8 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/client" - "github.com/kelseyhightower/envconfig" "github.com/leekchan/timeutil" + "github.com/offen/envconfig" "github.com/otiai10/copy" "golang.org/x/sync/errgroup" ) @@ -89,6 +89,28 @@ func newScript() (*script, error) { return nil }) + envconfig.Lookup = func(key string) (string, bool) { + value, okValue := os.LookupEnv(key) + location, okFile := os.LookupEnv(key + "_FILE") + + switch { + case okValue && !okFile: // only value + return value, true + case !okValue && okFile: // only file + contents, err := os.ReadFile(location) + if err != nil { + s.must(fmt.Errorf("newScript: failed to read %s! Error: %s", location, err)) + return "", false + } + return string(contents), true + case okValue && okFile: // both + s.must(fmt.Errorf("newScript: both %s and %s are set!", key, key+"_FILE")) + return "", false + default: // neither, ignore + return "", false + } + } + if err := envconfig.Process("", s.c); err != nil { return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err) } @@ -137,18 +159,10 @@ func newScript() (*script, error) { } if s.c.AwsS3BucketName != "" { - accessKeyID, err := s.c.resolveSecret(s.c.AwsAccessKeyID, s.c.AwsAccessKeyIDFile) - if err != nil { - return nil, fmt.Errorf("newScript: error resolving AwsAccessKeyID: %w", err) - } - secretAccessKey, err := s.c.resolveSecret(s.c.AwsSecretAccessKey, s.c.AwsSecretAccessKeyFile) - if err != nil { - return nil, fmt.Errorf("newScript: error resolving AwsSecretAccessKey: %w", err) - } s3Config := s3.Config{ Endpoint: s.c.AwsEndpoint, - AccessKeyID: accessKeyID, - SecretAccessKey: secretAccessKey, + AccessKeyID: s.c.AwsAccessKeyID, + SecretAccessKey: s.c.AwsSecretAccessKey, IamRoleEndpoint: s.c.AwsIamRoleEndpoint, EndpointProto: s.c.AwsEndpointProto, EndpointInsecure: s.c.AwsEndpointInsecure, diff --git a/go.mod b/go.mod index 08d739e..ff5a987 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,10 @@ require ( github.com/cosiner/argv v0.1.0 github.com/docker/docker v24.0.5+incompatible github.com/gofrs/flock v0.8.1 - github.com/kelseyhightower/envconfig v1.4.0 github.com/klauspost/compress v1.16.7 github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d github.com/minio/minio-go/v7 v7.0.62 + github.com/offen/envconfig v1.5.0 github.com/otiai10/copy v1.11.0 github.com/pkg/sftp v1.13.6 github.com/studio-b12/gowebdav v0.9.0 diff --git a/go.sum b/go.sum index b66a019..8b3587b 100644 --- a/go.sum +++ b/go.sum @@ -451,8 +451,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= -github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= @@ -527,6 +525,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/offen/envconfig v1.5.0 h1:LHL4wYIDVeoGxSDI40MShmWfss3gYUlCdstfSiSq4Fk= +github.com/offen/envconfig v1.5.0/go.mod h1:L7ny7R+4JWH3VVnZ+ARHvZysWUiZ2eQcm3L0imU9ACY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= diff --git a/test/secrets/run.sh b/test/secrets/run.sh index aae0cd6..860daf5 100755 --- a/test/secrets/run.sh +++ b/test/secrets/run.sh @@ -31,6 +31,16 @@ pass "Found relevant files in untared backup." sleep 5 expect_running_containers "5" +docker exec -e AWS_ACCESS_KEY_ID=test $(docker ps -q -f name=backup) backup \ + && fail "Backup should have failed due to duplicate env variables." + +pass "Backup failed due to duplicate env variables." + +docker exec -e AWS_ACCESS_KEY_ID_FILE=/tmp/nonexistant $(docker ps -q -f name=backup) backup \ + && fail "Backup should have failed due to non existing file env variable." + +pass "Backup failed due to non existing file env variable." + docker stack rm test_stack docker secret rm minio_root_password diff --git a/test/swarm/run.sh b/test/swarm/run.sh index 4028cc5..69fddca 100755 --- a/test/swarm/run.sh +++ b/test/swarm/run.sh @@ -29,6 +29,7 @@ sleep 5 expect_running_containers "5" docker stack rm test_stack +sleep 1 docker swarm leave --force sleep 10