Env vars should propagate when using conf.d (#358)

* Extend confd test case to test for env var propagation

* Env vars set in conf.d files are expected to propagate

* Lock needs to be acquired when instantiating script
This commit is contained in:
Frederik Ring 2024-02-13 15:43:04 +01:00 committed by GitHub
parent 241b5d2f25
commit 9a1e885138
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 68 additions and 42 deletions

View File

@ -49,8 +49,9 @@ func loadEnvVars() (*Config, error) {
} }
type configFile struct { type configFile struct {
name string name string
config *Config config *Config
additionalEnvVars map[string]string
} }
func loadEnvFiles(directory string) ([]configFile, error) { func loadEnvFiles(directory string) ([]configFile, error) {
@ -74,13 +75,16 @@ func loadEnvFiles(directory string) ([]configFile, error) {
} }
lookup := func(key string) (string, bool) { lookup := func(key string) (string, bool) {
val, ok := envFile[key] val, ok := envFile[key]
return val, ok if ok {
return val, ok
}
return os.LookupEnv(key)
} }
c, err := loadConfig(lookup) c, err := loadConfig(lookup)
if err != nil { if err != nil {
return nil, fmt.Errorf("loadEnvFiles: error loading config from file %s: %w", p, err) return nil, fmt.Errorf("loadEnvFiles: error loading config from file %s: %w", p, err)
} }
cs = append(cs, configFile{config: c, name: item.Name()}) cs = append(cs, configFile{config: c, name: item.Name(), additionalEnvVars: envFile})
} }
return cs, nil return cs, nil

View File

@ -36,33 +36,26 @@ func (c *command) must(err error) {
} }
} }
func runScript(c *Config) (err error) { func runScript(c *Config, envVars map[string]string) (err error) {
defer func() { defer func() {
if derr := recover(); derr != nil { if derr := recover(); derr != nil {
err = fmt.Errorf("runScript: unexpected panic running script: %v", err) err = fmt.Errorf("runScript: unexpected panic running script: %v", err)
} }
}() }()
s, err := newScript(c) s, unlock, err := newScript(c, envVars)
if err != nil { if err != nil {
err = fmt.Errorf("runScript: error instantiating script: %w", err) err = fmt.Errorf("runScript: error instantiating script: %w", err)
return return
} }
defer func() {
derr := unlock()
if err != nil {
err = fmt.Errorf("runScript: error releasing file lock: %w", derr)
}
}()
runErr := func() (err error) { runErr := func() (err error) {
unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
if err != nil {
err = fmt.Errorf("runScript: error acquiring file lock: %w", err)
return
}
defer func() {
derr := unlock()
if err == nil && derr != nil {
err = fmt.Errorf("runScript: error releasing file lock: %w", derr)
}
}()
scriptErr := func() error { scriptErr := func() error {
if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) { if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) {
restartContainersAndServices, err := s.stopContainersAndServices() restartContainersAndServices, err := s.stopContainersAndServices()
@ -132,7 +125,7 @@ func (c *command) runInForeground(profileCronExpression string) error {
), ),
) )
addJob := func(config *Config, name string) error { addJob := func(config *Config, name string, envVars map[string]string) error {
if _, err := cr.AddFunc(config.BackupCronExpression, func() { if _, err := cr.AddFunc(config.BackupCronExpression, func() {
c.logger.Info( c.logger.Info(
fmt.Sprintf( fmt.Sprintf(
@ -140,7 +133,8 @@ func (c *command) runInForeground(profileCronExpression string) error {
config.BackupCronExpression, config.BackupCronExpression,
), ),
) )
if err := runScript(config); err != nil {
if err := runScript(config, envVars); err != nil {
c.logger.Error( c.logger.Error(
fmt.Sprintf( fmt.Sprintf(
"Unexpected error running schedule %s: %v", "Unexpected error running schedule %s: %v",
@ -175,7 +169,7 @@ func (c *command) runInForeground(profileCronExpression string) error {
if err != nil { if err != nil {
return fmt.Errorf("runInForeground: could not load config from environment variables: %w", err) return fmt.Errorf("runInForeground: could not load config from environment variables: %w", err)
} else { } else {
err = addJob(c, "from environment") err = addJob(c, "from environment", nil)
if err != nil { if err != nil {
return fmt.Errorf("runInForeground: error adding job from env: %w", err) return fmt.Errorf("runInForeground: error adding job from env: %w", err)
} }
@ -183,7 +177,7 @@ func (c *command) runInForeground(profileCronExpression string) error {
} else { } else {
c.logger.Info("/etc/dockervolumebackup/conf.d was found, using configuration files from this directory.") c.logger.Info("/etc/dockervolumebackup/conf.d was found, using configuration files from this directory.")
for _, config := range cs { for _, config := range cs {
err = addJob(config.config, config.name) err = addJob(config.config, config.name, config.additionalEnvVars)
if err != nil { if err != nil {
return fmt.Errorf("runInForeground: error adding jobs from conf files: %w", err) return fmt.Errorf("runInForeground: error adding jobs from conf files: %w", err)
} }
@ -227,7 +221,7 @@ func (c *command) runAsCommand() error {
if err != nil { if err != nil {
return fmt.Errorf("runAsCommand: error loading env vars: %w", err) return fmt.Errorf("runAsCommand: error loading env vars: %w", err)
} }
err = runScript(config) err = runScript(config, nil)
if err != nil { if err != nil {
return fmt.Errorf("runAsCommand: error running script: %w", err) return fmt.Errorf("runAsCommand: error running script: %w", err)
} }

View File

@ -57,7 +57,7 @@ type script struct {
// remote resources like the Docker engine or remote storage locations. All // remote resources like the Docker engine or remote storage locations. All
// reading from env vars or other configuration sources is expected to happen // reading from env vars or other configuration sources is expected to happen
// in this method. // in this method.
func newScript(c *Config) (*script, error) { func newScript(c *Config, envVars map[string]string) (*script, func() error, error) {
stdOut, logBuffer := buffer(os.Stdout) stdOut, logBuffer := buffer(os.Stdout)
s := &script{ s := &script{
c: c, c: c,
@ -76,6 +76,31 @@ func newScript(c *Config) (*script, error) {
}, },
} }
unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
if err != nil {
return nil, noop, fmt.Errorf("runScript: error acquiring file lock: %w", err)
}
for key, value := range envVars {
currentVal, currentOk := os.LookupEnv(key)
defer func(currentKey, currentVal string, currentOk bool) {
if !currentOk {
_ = os.Unsetenv(currentKey)
} else {
_ = os.Setenv(currentKey, currentVal)
}
s.logger.Info(fmt.Sprintf("unset %v: %v", currentKey, currentVal))
}(key, currentVal, currentOk)
if err := os.Setenv(key, value); err != nil {
return nil, unlock, fmt.Errorf(
"Unexpected error overloading environment %s: %w",
s.c.BackupCronExpression,
err,
)
}
s.logger.Info(fmt.Sprintf("set %v: %v", key, value))
}
s.registerHook(hookLevelPlumbing, func(error) error { s.registerHook(hookLevelPlumbing, func(error) error {
s.stats.EndTime = time.Now() s.stats.EndTime = time.Now()
s.stats.TookTime = s.stats.EndTime.Sub(s.stats.StartTime) s.stats.TookTime = s.stats.EndTime.Sub(s.stats.StartTime)
@ -86,14 +111,14 @@ func newScript(c *Config) (*script, error) {
tmplFileName, tErr := template.New("extension").Parse(s.file) tmplFileName, tErr := template.New("extension").Parse(s.file)
if tErr != nil { if tErr != nil {
return nil, fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr) return nil, unlock, fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr)
} }
var bf bytes.Buffer var bf bytes.Buffer
if tErr := tmplFileName.Execute(&bf, map[string]string{ if tErr := tmplFileName.Execute(&bf, map[string]string{
"Extension": fmt.Sprintf("tar.%s", s.c.BackupCompression), "Extension": fmt.Sprintf("tar.%s", s.c.BackupCompression),
}); tErr != nil { }); tErr != nil {
return nil, fmt.Errorf("newScript: error executing backup file extension template: %w", tErr) return nil, unlock, fmt.Errorf("newScript: error executing backup file extension template: %w", tErr)
} }
s.file = bf.String() s.file = bf.String()
@ -104,12 +129,12 @@ func newScript(c *Config) (*script, error) {
} }
s.file = timeutil.Strftime(&s.stats.StartTime, s.file) s.file = timeutil.Strftime(&s.stats.StartTime, s.file)
_, err := os.Stat("/var/run/docker.sock") _, err = os.Stat("/var/run/docker.sock")
_, dockerHostSet := os.LookupEnv("DOCKER_HOST") _, dockerHostSet := os.LookupEnv("DOCKER_HOST")
if !os.IsNotExist(err) || dockerHostSet { if !os.IsNotExist(err) || dockerHostSet {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil { if err != nil {
return nil, fmt.Errorf("newScript: failed to create docker client") return nil, unlock, fmt.Errorf("newScript: failed to create docker client")
} }
s.cli = cli s.cli = cli
s.registerHook(hookLevelPlumbing, func(err error) error { s.registerHook(hookLevelPlumbing, func(err error) error {
@ -147,7 +172,7 @@ func newScript(c *Config) (*script, error) {
} }
s3Backend, err := s3.NewStorageBackend(s3Config, logFunc) s3Backend, err := s3.NewStorageBackend(s3Config, logFunc)
if err != nil { if err != nil {
return nil, fmt.Errorf("newScript: error creating s3 storage backend: %w", err) return nil, unlock, fmt.Errorf("newScript: error creating s3 storage backend: %w", err)
} }
s.storages = append(s.storages, s3Backend) s.storages = append(s.storages, s3Backend)
} }
@ -162,7 +187,7 @@ func newScript(c *Config) (*script, error) {
} }
webdavBackend, err := webdav.NewStorageBackend(webDavConfig, logFunc) webdavBackend, err := webdav.NewStorageBackend(webDavConfig, logFunc)
if err != nil { if err != nil {
return nil, fmt.Errorf("newScript: error creating webdav storage backend: %w", err) return nil, unlock, fmt.Errorf("newScript: error creating webdav storage backend: %w", err)
} }
s.storages = append(s.storages, webdavBackend) s.storages = append(s.storages, webdavBackend)
} }
@ -179,7 +204,7 @@ func newScript(c *Config) (*script, error) {
} }
sshBackend, err := ssh.NewStorageBackend(sshConfig, logFunc) sshBackend, err := ssh.NewStorageBackend(sshConfig, logFunc)
if err != nil { if err != nil {
return nil, fmt.Errorf("newScript: error creating ssh storage backend: %w", err) return nil, unlock, fmt.Errorf("newScript: error creating ssh storage backend: %w", err)
} }
s.storages = append(s.storages, sshBackend) s.storages = append(s.storages, sshBackend)
} }
@ -203,7 +228,7 @@ func newScript(c *Config) (*script, error) {
} }
azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
if err != nil { if err != nil {
return nil, fmt.Errorf("newScript: error creating azure storage backend: %w", err) return nil, unlock, fmt.Errorf("newScript: error creating azure storage backend: %w", err)
} }
s.storages = append(s.storages, azureBackend) s.storages = append(s.storages, azureBackend)
} }
@ -220,7 +245,7 @@ func newScript(c *Config) (*script, error) {
} }
dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc) dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc)
if err != nil { if err != nil {
return nil, fmt.Errorf("newScript: error creating dropbox storage backend: %w", err) return nil, unlock, fmt.Errorf("newScript: error creating dropbox storage backend: %w", err)
} }
s.storages = append(s.storages, dropboxBackend) s.storages = append(s.storages, dropboxBackend)
} }
@ -246,14 +271,14 @@ func newScript(c *Config) (*script, error) {
hookLevel, ok := hookLevels[s.c.NotificationLevel] hookLevel, ok := hookLevels[s.c.NotificationLevel]
if !ok { if !ok {
return nil, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel) return nil, unlock, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel)
} }
s.hookLevel = hookLevel s.hookLevel = hookLevel
if len(s.c.NotificationURLs) > 0 { if len(s.c.NotificationURLs) > 0 {
sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...) sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...)
if senderErr != nil { if senderErr != nil {
return nil, fmt.Errorf("newScript: error creating sender: %w", senderErr) return nil, unlock, fmt.Errorf("newScript: error creating sender: %w", senderErr)
} }
s.sender = sender s.sender = sender
@ -261,13 +286,13 @@ func newScript(c *Config) (*script, error) {
tmpl.Funcs(templateHelpers) tmpl.Funcs(templateHelpers)
tmpl, err = tmpl.Parse(defaultNotifications) tmpl, err = tmpl.Parse(defaultNotifications)
if err != nil { if err != nil {
return nil, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err) return nil, unlock, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
} }
if fi, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil && fi.IsDir() { if fi, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil && fi.IsDir() {
tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*") tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*")
if err != nil { if err != nil {
return nil, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err) return nil, unlock, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
} }
} }
s.template = tmpl s.template = tmpl
@ -288,7 +313,7 @@ func newScript(c *Config) (*script, error) {
}) })
} }
return s, nil return s, unlock, nil
} }
// createArchive creates a tar archive of the configured backup location and // createArchive creates a tar archive of the configured backup location and

View File

@ -1,2 +1,2 @@
BACKUP_FILENAME="conf.tar.gz" NAME="conf"
BACKUP_CRON_EXPRESSION="*/1 * * * *" BACKUP_CRON_EXPRESSION="*/1 * * * *"

View File

@ -1,2 +1,2 @@
BACKUP_FILENAME="other.tar.gz" NAME="other"
BACKUP_CRON_EXPRESSION="*/1 * * * *" BACKUP_CRON_EXPRESSION="*/1 * * * *"

View File

@ -1,2 +1,2 @@
BACKUP_FILENAME="never.tar.gz" NAME="never"
BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?" BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?"

View File

@ -4,6 +4,9 @@ services:
backup: backup:
image: offen/docker-volume-backup:${TEST_VERSION:-canary} image: offen/docker-volume-backup:${TEST_VERSION:-canary}
restart: always restart: always
environment:
BACKUP_FILENAME: $$NAME.tar.gz
BACKUP_FILENAME_EXPAND: 'true'
volumes: volumes:
- ${LOCAL_DIR:-./local}:/archive - ${LOCAL_DIR:-./local}:/archive
- app_data:/backup/app_data:ro - app_data:/backup/app_data:ro