From 749ca85db913e077738c0663dc07b024e1eab515 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Wed, 7 Feb 2024 19:24:05 +0100 Subject: [PATCH] Hoist control for exiting script a level up --- cmd/backup/config_provider.go | 6 +- cmd/backup/exec.go | 4 +- cmd/backup/main.go | 233 ++++++++++++++++++++-------------- cmd/backup/script.go | 11 -- test/confd/run.sh | 2 + 5 files changed, 143 insertions(+), 113 deletions(-) diff --git a/cmd/backup/config_provider.go b/cmd/backup/config_provider.go index 424afab..8f8103a 100644 --- a/cmd/backup/config_provider.go +++ b/cmd/backup/config_provider.go @@ -54,7 +54,7 @@ func loadEnvFiles(directory string) ([]*Config, error) { if os.IsNotExist(err) { return nil, err } - return nil, fmt.Errorf("failed to read files from env directory, error: %w", err) + return nil, fmt.Errorf("loadEnvFiles: failed to read files from env directory: %w", err) } var cs = make([]*Config, 0) @@ -63,7 +63,7 @@ func loadEnvFiles(directory string) ([]*Config, error) { p := filepath.Join(directory, item.Name()) envFile, err := godotenv.Read(p) if err != nil { - return nil, fmt.Errorf("error reading config file %s, error: %w", p, err) + return nil, fmt.Errorf("loadEnvFiles: error reading config file %s: %w", p, err) } lookup := func(key string) (string, bool) { val, ok := envFile[key] @@ -71,7 +71,7 @@ func loadEnvFiles(directory string) ([]*Config, error) { } c, err := loadConfig(lookup) if err != nil { - return nil, fmt.Errorf("error loading config from file %s, error: %w", p, err) + return nil, fmt.Errorf("loadEnvFiles: error loading config from file %s: %w", p, err) } cs = append(cs, c) } diff --git a/cmd/backup/exec.go b/cmd/backup/exec.go index 0f4f7b5..06692dc 100644 --- a/cmd/backup/exec.go +++ b/cmd/backup/exec.go @@ -188,12 +188,12 @@ func (s *script) withLabeledCommands(step lifecyclePhase, cb func() error) func( if s.cli == nil { return cb } - return func() error { + return func() (ret error) { if err := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil { return fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err) } defer func() { - s.must(s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step))) + ret = s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step)) }() return cb() } diff --git a/cmd/backup/main.go b/cmd/backup/main.go index c10b9b1..7bd59bc 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -14,79 +14,109 @@ import ( "github.com/robfig/cron/v3" ) -func runScript(c *Config) (ret error) { - s, err := newScript(c) - if err != nil { - return err - } - - unlock, err := s.lock("/var/lock/dockervolumebackup.lock") - if err != nil { - return err - } - - defer func() { - err = unlock() - if err != nil { - ret = err - } - }() - - defer func() { - if pArg := recover(); pArg != nil { - if err, ok := pArg.(error); ok { - s.logger.Error( - fmt.Sprintf("Executing the script encountered a panic: %v", err), - ) - if hookErr := s.runHooks(err); hookErr != nil { - s.logger.Error( - fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr), - ) - } - ret = err - } else { - s.logger.Error( - fmt.Sprintf("Executing the script encountered an unrecoverable panic: %v", err), - ) - - panic(pArg) - } - } - - if err := s.runHooks(nil); err != nil { - s.logger.Error( - fmt.Sprintf( - "Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v", - err, - ), - ) - ret = err - } - s.logger.Info("Finished running backup tasks.") - }() - - s.must(s.withLabeledCommands(lifecyclePhaseArchive, func() error { - restartContainersAndServices, err := s.stopContainersAndServices() - // The mechanism for restarting containers is not using hooks as it - // should happen as soon as possible (i.e. before uploading backups or - // similar). - defer func() { - s.must(restartContainersAndServices()) - }() - if err != nil { - return err - } - return s.createArchive() - })()) - - s.must(s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)()) - s.must(s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)()) - s.must(s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)()) - - return nil +type command struct { + logger *slog.Logger } -func runInForeground() error { +func newCommand() *command { + return &command{ + logger: slog.New(slog.NewTextHandler(os.Stdout, nil)), + } +} + +func (c *command) must(err error) { + if err != nil { + c.logger.Error( + fmt.Sprintf("Fatal error running command: %v", err), + "error", + err, + ) + os.Exit(1) + } +} + +func runScript(c *Config) (ret error) { + defer func() { + if err := recover(); err != nil { + ret = fmt.Errorf("runScript: unexpected panic running script: %v", err) + } + }() + + s, err := newScript(c) + if err != nil { + return fmt.Errorf("runScript: error instantiating script: %w", err) + } + + runErr := func() (ret error) { + unlock, err := s.lock("/var/lock/dockervolumebackup.lock") + if err != nil { + return fmt.Errorf("runScript: error acquiring file lock: %w", err) + } + + defer func() { + err = unlock() + if err != nil { + ret = fmt.Errorf("runScript: error releasing file lock: %w", err) + } + }() + + scriptErr := func() error { + if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (ret error) { + restartContainersAndServices, err := s.stopContainersAndServices() + // The mechanism for restarting containers is not using hooks as it + // should happen as soon as possible (i.e. before uploading backups or + // similar). + defer func() { + ret = restartContainersAndServices() + }() + if err != nil { + return err + } + return s.createArchive() + })(); err != nil { + return err + } + + if err := s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)(); err != nil { + return err + } + if err := s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)(); err != nil { + return err + } + if err := s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)(); err != nil { + return err + } + return nil + }() + if hookErr := s.runHooks(scriptErr); hookErr != nil { + if scriptErr != nil { + return fmt.Errorf( + "runScript: error %w executing the script followed by %w calling the registered hooks", + scriptErr, + hookErr, + ) + } + return fmt.Errorf( + "runScript: the script ran successfully, but an error occurred calling the registered hooks: %w", + hookErr, + ) + } + if scriptErr != nil { + return fmt.Errorf("runScript: error running script: %w", err) + } + return nil + }() + + if runErr != nil { + s.logger.Error( + fmt.Sprintf("Script run failed: %v", runErr), "error", runErr, + ) + } + return runErr + +} + +func (c *command) runInForeground() error { cr := cron.New( cron.WithParser( cron.NewParser( @@ -95,37 +125,51 @@ func runInForeground() error { ), ) - addJob := func(c *Config) error { - _, err := cr.AddFunc(c.BackupCronExpression, func() { - err := runScript(c) - if err != nil { - slog.Error("unexpected error during backup", "error", err) + addJob := func(config *Config) error { + if _, err := cr.AddFunc(config.BackupCronExpression, func() { + if err := runScript(config); err != nil { + c.logger.Error( + fmt.Sprintf( + "Unexpected error running schedule %v: %v", + config.BackupCronExpression, + err, + ), + "error", + err, + ) } - }) - return err + }); err != nil { + return fmt.Errorf("addJob: error adding schedule %s: %w", config.BackupCronExpression, err) + } + c.logger.Info(fmt.Sprintf("Successfully scheduled backup with expression %s", config.BackupCronExpression)) + return nil } cs, err := loadEnvFiles("/etc/dockervolumebackup/conf.d") if err != nil { if !os.IsNotExist(err) { - return fmt.Errorf("could not load config from environment files, error: %w", err) + return fmt.Errorf("runInForeground: could not load config from environment files: %w", err) } c, err := loadEnvVars() if err != nil { - return fmt.Errorf("could not load config from environment variables") + return fmt.Errorf("runInForeground: could not load config from environment variables: %w", err) } else { err = addJob(c) if err != nil { - return fmt.Errorf("could not add cron job, error: %w", err) + return fmt.Errorf("runInForeground: error adding job from env: %w", err) } } } else { - for _, c := range cs { - err = addJob(c) + c.logger.Info("/etc/dockervolumebackup/conf.d was found, using configuration files from this directory.") + for _, config := range cs { + err = addJob(config) if err != nil { - return fmt.Errorf("could not add cron job, error: %w", err) + return fmt.Errorf("runInForeground: error adding jobs from conf files: %w", err) } + c.logger.Info( + fmt.Sprintf("Successfully scheduled backup with expression %s", config.BackupCronExpression), + ) } } @@ -139,13 +183,13 @@ func runInForeground() error { return nil } -func runAsCommand() error { - c, err := loadEnvVars() +func (c *command) runAsCommand() error { + config, err := loadEnvVars() if err != nil { return fmt.Errorf("could not load config from environment variables, error: %w", err) } - err = runScript(c) + err = runScript(config) if err != nil { return fmt.Errorf("unexpected error during backup, error: %w", err) } @@ -154,18 +198,13 @@ func runAsCommand() error { } func main() { - serve := flag.Bool("foreground", false, "run the tool in the foreground") + foreground := flag.Bool("foreground", false, "run the tool in the foreground") flag.Parse() - var err error - if *serve { - err = runInForeground() + c := newCommand() + if *foreground { + c.must(c.runInForeground()) } else { - err = runAsCommand() - } - - if err != nil { - slog.Error("ran into an issue during execution", "error", err) - os.Exit(1) + c.must(c.runAsCommand()) } } diff --git a/cmd/backup/script.go b/cmd/backup/script.go index 324f5a0..fa9719f 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -480,17 +480,6 @@ func (s *script) pruneBackups() error { return nil } -// must exits the script run prematurely in case the given error -// is non-nil. -func (s *script) must(err error) { - if err != nil { - s.logger.Error( - fmt.Sprintf("Fatal error running backup: %s", err), - ) - panic(err) - } -} - // skipPrune returns true if the given backend name is contained in the // list of skipped backends. func skipPrune(name string, skippedBackends []string) bool { diff --git a/test/confd/run.sh b/test/confd/run.sh index 3a5fca9..f81407a 100755 --- a/test/confd/run.sh +++ b/test/confd/run.sh @@ -13,6 +13,8 @@ docker compose up -d --quiet-pull # sleep until a backup is guaranteed to have happened on the 1 minute schedule sleep 100 +docker compose logs backup + if [ ! -f "$LOCAL_DIR/conf.tar.gz" ]; then fail "Config from file was not used." fi