From 2c06f8150370ab1e53633f94815ecddf9d00de91 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 9 Sep 2021 07:24:18 +0200 Subject: [PATCH 1/3] collect all log output in buffer so it could be used in notifications --- cmd/backup/main.go | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/cmd/backup/main.go b/cmd/backup/main.go index 21bf028..cb82d05 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "context" "errors" "fmt" @@ -62,8 +63,9 @@ type script struct { mc *minio.Client logger *logrus.Logger - start time.Time - file string + start time.Time + file string + output *bytes.Buffer c *config } @@ -90,15 +92,17 @@ type config struct { // reading from env vars or other configuration sources is expected to happen // in this method. func newScript() (*script, error) { + stdOut, logBuffer := buffer(os.Stdout) s := &script{ c: &config{}, logger: &logrus.Logger{ - Out: os.Stdout, + Out: stdOut, Formatter: new(logrus.TextFormatter), Hooks: make(logrus.LevelHooks), Level: logrus.InfoLevel, }, - start: time.Now(), + start: time.Now(), + output: logBuffer, } if err := envconfig.Process("", s.c); err != nil { @@ -526,3 +530,22 @@ func join(errs ...error) error { } return errors.New("[" + strings.Join(msgs, ", ") + "]") } + +// buffer takes an io.Writer and returns a wrapped version of the +// writer that writes to both the original target as well as the returned buffer +func buffer(w io.Writer) (io.Writer, *bytes.Buffer) { + buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w} + return buffering, &buffering.buf +} + +type bufferingWriter struct { + buf bytes.Buffer + writer io.Writer +} + +func (b *bufferingWriter) Write(p []byte) (n int, err error) { + if n, err := b.buf.Write(p); err != nil { + return n, fmt.Errorf("bufferingWriter: error writing to buffer: %w", err) + } + return b.writer.Write(p) +} From e46968ed794da520cca73008d3b1287ff5faf95a Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 9 Sep 2021 08:12:07 +0200 Subject: [PATCH 2/3] call error hooks on script failure --- cmd/backup/main.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cmd/backup/main.go b/cmd/backup/main.go index cb82d05..5f05ca0 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -59,9 +59,10 @@ func main() { // script holds all the stateful information required to orchestrate a // single backup run. type script struct { - cli *client.Client - mc *minio.Client - logger *logrus.Logger + cli *client.Client + mc *minio.Client + logger *logrus.Logger + errorHooks []errorHook start time.Time file string @@ -70,6 +71,8 @@ type script struct { c *config } +type errorHook func(err error, start time.Time, logOutput string) error + type config struct { BackupSources string `split_words:"true" default:"/backup"` BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` @@ -473,9 +476,15 @@ func (s *script) pruneOldBackups() error { } // must exits the script run non-zero and prematurely in case the given error -// is non-nil. +// is non-nil. If error hooks are present on the script object, they +// will be called, passing the failure and previous log output. func (s *script) must(err error) { if err != nil { + for _, hook := range s.errorHooks { + if hookErr := hook(err, s.start, s.output.String()); hookErr != nil { + s.logger.Errorf("An error occurred calling an error hook: %s", err) + } + } s.logger.Fatalf("Fatal error running backup: %s", err) } } From 88368197c1ab4bd859b6a142fefbfbb0e770ad64 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Thu, 9 Sep 2021 08:58:03 +0200 Subject: [PATCH 3/3] implement email notifications on failed backup runs --- README.md | 56 +++++++++++++++++++++++++++++++++++++++++++ cmd/backup/main.go | 59 ++++++++++++++++++++++++++++++++++------------ go.mod | 2 ++ go.sum | 4 ++++ 4 files changed, 106 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 7e387ae..cf3e724 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc - [How to](#how-to) - [Stopping containers during backup](#stopping-containers-during-backup) - [Automatically pruning old backups](#automatically-pruning-old-backups) + - [Send email notifications on failed backup runs](#send-email-notifications-on-failed-backup-runs) - [Encrypting your backup using GPG](#encrypting-your-backup-using-gpg) - [Restoring a volume from a backup](#restoring-a-volume-from-a-backup) - [Using with Docker Swarm](#using-with-docker-swarm) @@ -55,6 +56,10 @@ services: - docker-volume-backup.stop-during-backup=true backup: + # In production, it is advised to lock your image tag to a proper + # release version instead of using `latest`. + # Check https://github.com/offen/docker-volume-backup/releases + # for a list of available releases. image: offen/docker-volume-backup:latest restart: always env_file: ./backup.env # see below for configuration reference @@ -188,6 +193,38 @@ You can populate below template according to your requirements and use it as you # override this default by specifying a different value here. # BACKUP_STOP_CONTAINER_LABEL="service1" + +########### EMAIL NOTIFICATIONS ON FAILED BACKUP RUNS + +# In case SMTP credentials are provided, notification emails can be sent out on +# failed backup runs. These emails will contain the start time, the error +# message and all log output prior to the failure. + +# The recipient(s) of the notification. Supply a comma separated list +# of adresses if you want to notify multiple recipients. If this is +# not set, no emails will be sent. + +# EMAIL_NOTIFICATION_RECIPIENT="you@example.com" + +# The "From" header of the sent email. Defaults to `noreply@nohost`. + +# EMAIL_NOTIFICATION_SENDER="no-reply@example.com" + +# The hostname of your SMTP server. + +# EMAIL_SMTP_HOST="posteo.de" + +# The SMTP password. + +# EMAIL_SMTP_PASSWORD="" + +# The SMTP username. + +# EMAIL_SMTP_USERNAME="no-reply@example.com" + +The port used when communicating with the server. Defaults to 587. + +# EMAIL_SMTP_PORT="" ``` ## How to @@ -247,6 +284,25 @@ volumes: data: ``` +### Send email notifications on failed backup runs + +To send out email notifications on failed backup runs, provide SMTP credentials, a sender and a recipient: + +```yml +version: '3' + +services: + backup: + image: offen/docker-volume-backup:latest + environment: + # ... other configuration values go here + EMAIL_SMTP_HOST: "smtp.example.com" + EMAIL_SMTP_PASSWORD: "password" + EMAIL_SMTP_USERNAME: "username" + EMAIL_NOTIFICATION_SENDER: "noreply@example.com" + EMAIL_NOTIFICATION_RECIPIENT: "notifications@example.com" +``` + ### Encrypting your backup using GPG The image supports encrypting backups using GPG out of the box. diff --git a/cmd/backup/main.go b/cmd/backup/main.go index 5f05ca0..cc7dd5e 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/swarm" "github.com/docker/docker/client" + "github.com/go-gomail/gomail" "github.com/gofrs/flock" "github.com/kelseyhightower/envconfig" "github.com/leekchan/timeutil" @@ -74,20 +75,26 @@ type script struct { type errorHook func(err error, start time.Time, logOutput string) error type config struct { - BackupSources string `split_words:"true" default:"/backup"` - BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` - BackupArchive string `split_words:"true" default:"/archive"` - BackupRetentionDays int32 `split_words:"true" default:"-1"` - BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` - BackupPruningPrefix string `split_words:"true"` - BackupStopContainerLabel string `split_words:"true" default:"true"` - AwsS3BucketName string `split_words:"true"` - AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` - AwsEndpointProto string `split_words:"true" default:"https"` - AwsEndpointInsecure bool `split_words:"true"` - AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` - AwsSecretAccessKey string `split_words:"true"` - GpgPassphrase string `split_words:"true"` + BackupSources string `split_words:"true" default:"/backup"` + BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` + BackupArchive string `split_words:"true" default:"/archive"` + BackupRetentionDays int32 `split_words:"true" default:"-1"` + BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` + BackupPruningPrefix string `split_words:"true"` + BackupStopContainerLabel string `split_words:"true" default:"true"` + AwsS3BucketName string `split_words:"true"` + AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` + AwsEndpointProto string `split_words:"true" default:"https"` + AwsEndpointInsecure bool `split_words:"true"` + AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` + AwsSecretAccessKey string `split_words:"true"` + GpgPassphrase string `split_words:"true"` + EmailNotificationRecipient string `split_words:"true"` + EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` + EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` + EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` + EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` + EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` } // newScript creates all resources needed for the script to perform actions against @@ -138,6 +145,28 @@ func newScript() (*script, error) { s.mc = mc } + if s.c.EmailNotificationRecipient != "" { + s.errorHooks = append(s.errorHooks, func(err error, start time.Time, logOutput string) error { + mailer := gomail.NewDialer( + s.c.EmailSMTPHost, s.c.EmailSMTPPort, s.c.EmailSMTPUsername, s.c.EmailSMTPPassword, + ) + + subject := fmt.Sprintf( + "Failure running docker-volume-backup at %s", start.Format(time.RFC3339), + ) + body := fmt.Sprintf( + "Running docker-volume-backup failed with error: %s\n\nLog output before the error occurred:\n\n%s\n", err, logOutput, + ) + + message := gomail.NewMessage() + message.SetHeader("From", s.c.EmailNotificationSender) + message.SetHeader("To", s.c.EmailNotificationRecipient) + message.SetHeader("Subject", subject) + message.SetBody("text/plain", body) + return mailer.DialAndSend(message) + }) + } + return s, nil } @@ -482,7 +511,7 @@ func (s *script) must(err error) { if err != nil { for _, hook := range s.errorHooks { if hookErr := hook(err, s.start, s.output.String()); hookErr != nil { - s.logger.Errorf("An error occurred calling an error hook: %s", err) + s.logger.Errorf("An error occurred calling an error hook: %s", hookErr) } } s.logger.Fatalf("Fatal error running backup: %s", err) diff --git a/go.mod b/go.mod index 30b1578..b07a5a6 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.0 // indirect github.com/google/uuid v1.2.0 // indirect @@ -41,5 +42,6 @@ require ( google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect google.golang.org/grpc v1.33.2 // indirect google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/ini.v1 v1.57.0 // indirect ) diff --git a/go.sum b/go.sum index ceca83c..babacaf 100644 --- a/go.sum +++ b/go.sum @@ -254,6 +254,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E= +github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -908,6 +910,8 @@ google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/l google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=