From 8bad0656b3e3f0c59a9ee10c0210271ec147e4f8 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Fri, 17 Dec 2021 17:44:22 +0100 Subject: [PATCH] Enable notifications on multiple levels --- README.md | 19 ++++++---- cmd/backup/main.go | 94 ++++++++++++++++++++++++++++------------------ 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 54bad0a..3f64a4d 100644 --- a/README.md +++ b/README.md @@ -236,9 +236,9 @@ You can populate below template according to your requirements and use it as you # BACKUP_STOP_CONTAINER_LABEL="service1" -########### NOTIFICATIONS ON FAILED BACKUP RUNS +########### NOTIFICATIONS -# In case a backup fails, notifications (email, Slack, etc.) can be sent out. +# Notifications (email, Slack, etc.) can be sent out when a backup run finishes. # Configuration is provided as a comma-separated list of URLs as consumed # by `shoutrrr`: https://containrrr.dev/shoutrrr/v0.5/services/overview/ # When providing multiple URLs or an URL that contains a comma, the values @@ -249,17 +249,22 @@ You can populate below template according to your requirements and use it as you # NOTIFICATION_URLS=smtp://username:password@host:587/?fromAddress=sender@example.com&toAddresses=recipient@example.com -########### EMAIL NOTIFICATIONS ON FAILED BACKUP RUNS +# By default, notifications will only be sent out when a backup run fails. +# To receive notifications for every run, set `NOTIFICATION_LEVEL` to `always` + +# NOTIFICATION_LEVEL="always" + +########### EMAIL NOTIFICATIONS # ************************************************************************ # Providing notification configuration like this has been deprecated -# and will be removed in the next major version. Please use NOTIFICATION_ULRS +# and will be removed in the next major version. Please use NOTIFICATION_URLS # as documented above instead. # ************************************************************************ -# In case SMTP credentials are provided, notification emails can be sent out on -# failed backup runs. These emails will contain the start time, the error -# message and all log output prior to the failure. +# In case SMTP credentials are provided, notification emails can be sent out when +# a backup run finished. These emails will contain the start time, the error +# message on failure and all prior log output. # The recipient(s) of the notification. Supply a comma separated list # of adresses if you want to notify multiple recipients. If this is diff --git a/cmd/backup/main.go b/cmd/backup/main.go index a8fdd47..b2aa03b 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -44,7 +44,7 @@ func main() { defer func() { if pArg := recover(); pArg != nil { if err, ok := pArg.(error); ok { - if hookErr := s.runHooks(err, hookLevelCleanup, hookLevelFailure); hookErr != nil { + if hookErr := s.runHooks(err, hookLevelCleanup, hookLevelFailure, hookLevelAlways); hookErr != nil { s.logger.Errorf("An error occurred calling the registered hooks: %s", hookErr) } os.Exit(1) @@ -52,7 +52,7 @@ func main() { panic(pArg) } - if err := s.runHooks(nil, hookLevelCleanup); err != nil { + if err := s.runHooks(nil, hookLevelCleanup, hookLevelAlways); err != nil { s.logger.Errorf( "Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v", err, @@ -115,6 +115,7 @@ type config struct { AwsIamRoleEndpoint string `split_words:"true"` GpgPassphrase string `split_words:"true"` NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` + NotificationLevel string `split_words:"true" default:"failure"` EmailNotificationRecipient string `split_words:"true"` EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` @@ -208,38 +209,19 @@ func newScript() (*script, error) { s.c.EmailNotificationRecipient, ) s.c.NotificationURLs = append(s.c.NotificationURLs, emailURL) - s.logger.Warn("Using EMAIL_* keys for providing notification configuration has been deprecated and will be removed in the next major version.") - s.logger.Warn("Please use NOTIFICATION_URLS instead. Refer to the README for an upgrade guide.") + s.logger.Warn( + "Using EMAIL_* keys for providing notification configuration has been deprecated and will be removed in the next major version.", + ) + s.logger.Warn( + "Please use NOTIFICATION_URLS instead. Refer to the README for an upgrade guide.", + ) } - if len(s.c.NotificationURLs) > 0 { - s.hooks = append(s.hooks, hook{hookLevelFailure, func(err error, start time.Time, logOutput string) error { - sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...) - if senderErr != nil { - return fmt.Errorf("notifications: error creating sender: %w", senderErr) - } - body := fmt.Sprintf( - "Running docker-volume-backup failed with error: %s\n\nLog output of the failed run was:\n\n%s\n", err, logOutput, - ) - - results := sender.Send(body, &sTypes.Params{ - "title": fmt.Sprintf( - "Failure running docker-volume-backup at %s", start.Format(time.RFC3339), - ), - }) - - var errs []error - for _, result := range results { - if result != nil { - errs = append(errs, result) - } - } - if len(errs) != 0 { - return fmt.Errorf("notifications: error sending message: %w", join(errs...)) - } - return nil - }}) + notificationLevel := hookLevelFailure + if s.c.NotificationLevel == "always" { + notificationLevel = hookLevelAlways } + s.registerHook(notificationLevel, s.sendNotification) return s, nil } @@ -247,10 +229,47 @@ func newScript() (*script, error) { var noop = func() error { return nil } // registerHook adds the given action at the given level. -func (s *script) registerHook(level hookLevel, action func(err error, start time.Time, logOutput string) error) { +func (s *script) registerHook(level hookLevel, action func(err error) error) { s.hooks = append(s.hooks, hook{level, action}) } +// sendNotification sends a notification to third party services, containing +// information about the result of a backup run +func (s *script) sendNotification(err error) error { + if len(s.c.NotificationURLs) == 0 { + return nil + } + + sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...) + if senderErr != nil { + return fmt.Errorf("notifications: error creating sender: %w", senderErr) + } + + var body, title string + if err != nil { + body = fmt.Sprintf( + "Running docker-volume-backup failed with error: %s\n\nLog output of the failed run was:\n\n%s\n", err, s.output.String(), + ) + title = fmt.Sprintf("Failure running docker-volume-backup at %s", s.start.Format(time.RFC3339)) + } else { + body = fmt.Sprintf( + "Running docker-volume-backup succeeded.\n\nLog output was:\n\n%s\n", s.output.String(), + ) + title = fmt.Sprintf("Success running docker-volume-backup at %s", s.start.Format(time.RFC3339)) + } + + var errs []error + for _, result := range sender.Send(body, &sTypes.Params{"title": title}) { + if result != nil { + errs = append(errs, result) + } + } + if len(errs) != 0 { + return fmt.Errorf("notifications: error sending message: %w", join(errs...)) + } + return nil +} + // stopContainers stops all Docker containers that are marked as to being // stopped during the backup and returns a function that can be called to // restart everything that has been stopped. @@ -373,7 +392,7 @@ func (s *script) takeBackup() error { if s.c.BackupFromSnapshot { backupSources = filepath.Join("/tmp", s.c.BackupSources) // copy before compressing guard against a situation where backup folder's content are still growing. - s.registerHook(hookLevelCleanup, func(error, time.Time, string) error { + s.registerHook(hookLevelCleanup, func(error) error { if err := remove(backupSources); err != nil { return fmt.Errorf("takeBackup: error removing snapshot: %w", err) } @@ -390,7 +409,7 @@ func (s *script) takeBackup() error { } tarFile := s.file - s.registerHook(hookLevelCleanup, func(error, time.Time, string) error { + s.registerHook(hookLevelCleanup, func(error) error { if err := remove(tarFile); err != nil { return fmt.Errorf("takeBackup: error removing tar file: %w", err) } @@ -414,7 +433,7 @@ func (s *script) encryptBackup() error { } gpgFile := fmt.Sprintf("%s.gpg", s.file) - s.registerHook(hookLevelCleanup, func(error, time.Time, string) error { + s.registerHook(hookLevelCleanup, func(error) error { if err := remove(gpgFile); err != nil { return fmt.Errorf("encryptBackup: error removing gpg file: %w", err) } @@ -648,7 +667,7 @@ func (s *script) runHooks(err error, levels ...hookLevel) error { if hook.level != level { continue } - if actionErr := hook.action(err, s.start, s.output.String()); actionErr != nil { + if actionErr := hook.action(err); actionErr != nil { actionErrors = append(actionErrors, fmt.Errorf("runHooks: error running hook: %w", actionErr)) } } @@ -762,12 +781,13 @@ func (b *bufferingWriter) Write(p []byte) (n int, err error) { // reaches a certain point (e.g. unsuccessful backup) type hook struct { level hookLevel - action func(err error, start time.Time, logOutput string) error + action func(err error) error } type hookLevel int const ( hookLevelFailure hookLevel = iota + hookLevelAlways hookLevelCleanup )