Merge pull request #24 from offen/failure-email

Enable sending out email notifications on failed backups
This commit is contained in:
Frederik Ring 2021-09-09 09:10:20 +02:00 committed by GitHub
commit 5d400cb943
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 145 additions and 22 deletions

View File

@ -12,6 +12,7 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
- [How to](#how-to) - [How to](#how-to)
- [Stopping containers during backup](#stopping-containers-during-backup) - [Stopping containers during backup](#stopping-containers-during-backup)
- [Automatically pruning old backups](#automatically-pruning-old-backups) - [Automatically pruning old backups](#automatically-pruning-old-backups)
- [Send email notifications on failed backup runs](#send-email-notifications-on-failed-backup-runs)
- [Encrypting your backup using GPG](#encrypting-your-backup-using-gpg) - [Encrypting your backup using GPG](#encrypting-your-backup-using-gpg)
- [Restoring a volume from a backup](#restoring-a-volume-from-a-backup) - [Restoring a volume from a backup](#restoring-a-volume-from-a-backup)
- [Using with Docker Swarm](#using-with-docker-swarm) - [Using with Docker Swarm](#using-with-docker-swarm)
@ -55,6 +56,10 @@ services:
- docker-volume-backup.stop-during-backup=true - docker-volume-backup.stop-during-backup=true
backup: backup:
# In production, it is advised to lock your image tag to a proper
# release version instead of using `latest`.
# Check https://github.com/offen/docker-volume-backup/releases
# for a list of available releases.
image: offen/docker-volume-backup:latest image: offen/docker-volume-backup:latest
restart: always restart: always
env_file: ./backup.env # see below for configuration reference env_file: ./backup.env # see below for configuration reference
@ -188,6 +193,38 @@ You can populate below template according to your requirements and use it as you
# override this default by specifying a different value here. # override this default by specifying a different value here.
# BACKUP_STOP_CONTAINER_LABEL="service1" # BACKUP_STOP_CONTAINER_LABEL="service1"
########### EMAIL NOTIFICATIONS ON FAILED BACKUP RUNS
# In case SMTP credentials are provided, notification emails can be sent out on
# failed backup runs. These emails will contain the start time, the error
# message and all log output prior to the failure.
# The recipient(s) of the notification. Supply a comma separated list
# of adresses if you want to notify multiple recipients. If this is
# not set, no emails will be sent.
# EMAIL_NOTIFICATION_RECIPIENT="you@example.com"
# The "From" header of the sent email. Defaults to `noreply@nohost`.
# EMAIL_NOTIFICATION_SENDER="no-reply@example.com"
# The hostname of your SMTP server.
# EMAIL_SMTP_HOST="posteo.de"
# The SMTP password.
# EMAIL_SMTP_PASSWORD="<xxx>"
# The SMTP username.
# EMAIL_SMTP_USERNAME="no-reply@example.com"
The port used when communicating with the server. Defaults to 587.
# EMAIL_SMTP_PORT="<port>"
``` ```
## How to ## How to
@ -247,6 +284,25 @@ volumes:
data: data:
``` ```
### Send email notifications on failed backup runs
To send out email notifications on failed backup runs, provide SMTP credentials, a sender and a recipient:
```yml
version: '3'
services:
backup:
image: offen/docker-volume-backup:latest
environment:
# ... other configuration values go here
EMAIL_SMTP_HOST: "smtp.example.com"
EMAIL_SMTP_PASSWORD: "password"
EMAIL_SMTP_USERNAME: "username"
EMAIL_NOTIFICATION_SENDER: "noreply@example.com"
EMAIL_NOTIFICATION_RECIPIENT: "notifications@example.com"
```
### Encrypting your backup using GPG ### Encrypting your backup using GPG
The image supports encrypting backups using GPG out of the box. The image supports encrypting backups using GPG out of the box.

View File

@ -4,6 +4,7 @@
package main package main
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"fmt" "fmt"
@ -18,6 +19,7 @@ import (
"github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm" "github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client" "github.com/docker/docker/client"
"github.com/go-gomail/gomail"
"github.com/gofrs/flock" "github.com/gofrs/flock"
"github.com/kelseyhightower/envconfig" "github.com/kelseyhightower/envconfig"
"github.com/leekchan/timeutil" "github.com/leekchan/timeutil"
@ -61,13 +63,17 @@ type script struct {
cli *client.Client cli *client.Client
mc *minio.Client mc *minio.Client
logger *logrus.Logger logger *logrus.Logger
errorHooks []errorHook
start time.Time start time.Time
file string file string
output *bytes.Buffer
c *config c *config
} }
type errorHook func(err error, start time.Time, logOutput string) error
type config struct { type config struct {
BackupSources string `split_words:"true" default:"/backup"` BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
@ -83,6 +89,12 @@ type config struct {
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsSecretAccessKey string `split_words:"true"` AwsSecretAccessKey string `split_words:"true"`
GpgPassphrase string `split_words:"true"` GpgPassphrase string `split_words:"true"`
EmailNotificationRecipient string `split_words:"true"`
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
} }
// newScript creates all resources needed for the script to perform actions against // newScript creates all resources needed for the script to perform actions against
@ -90,15 +102,17 @@ type config struct {
// reading from env vars or other configuration sources is expected to happen // reading from env vars or other configuration sources is expected to happen
// in this method. // in this method.
func newScript() (*script, error) { func newScript() (*script, error) {
stdOut, logBuffer := buffer(os.Stdout)
s := &script{ s := &script{
c: &config{}, c: &config{},
logger: &logrus.Logger{ logger: &logrus.Logger{
Out: os.Stdout, Out: stdOut,
Formatter: new(logrus.TextFormatter), Formatter: new(logrus.TextFormatter),
Hooks: make(logrus.LevelHooks), Hooks: make(logrus.LevelHooks),
Level: logrus.InfoLevel, Level: logrus.InfoLevel,
}, },
start: time.Now(), start: time.Now(),
output: logBuffer,
} }
if err := envconfig.Process("", s.c); err != nil { if err := envconfig.Process("", s.c); err != nil {
@ -131,6 +145,28 @@ func newScript() (*script, error) {
s.mc = mc s.mc = mc
} }
if s.c.EmailNotificationRecipient != "" {
s.errorHooks = append(s.errorHooks, func(err error, start time.Time, logOutput string) error {
mailer := gomail.NewDialer(
s.c.EmailSMTPHost, s.c.EmailSMTPPort, s.c.EmailSMTPUsername, s.c.EmailSMTPPassword,
)
subject := fmt.Sprintf(
"Failure running docker-volume-backup at %s", start.Format(time.RFC3339),
)
body := fmt.Sprintf(
"Running docker-volume-backup failed with error: %s\n\nLog output before the error occurred:\n\n%s\n", err, logOutput,
)
message := gomail.NewMessage()
message.SetHeader("From", s.c.EmailNotificationSender)
message.SetHeader("To", s.c.EmailNotificationRecipient)
message.SetHeader("Subject", subject)
message.SetBody("text/plain", body)
return mailer.DialAndSend(message)
})
}
return s, nil return s, nil
} }
@ -469,9 +505,15 @@ func (s *script) pruneOldBackups() error {
} }
// must exits the script run non-zero and prematurely in case the given error // must exits the script run non-zero and prematurely in case the given error
// is non-nil. // is non-nil. If error hooks are present on the script object, they
// will be called, passing the failure and previous log output.
func (s *script) must(err error) { func (s *script) must(err error) {
if err != nil { if err != nil {
for _, hook := range s.errorHooks {
if hookErr := hook(err, s.start, s.output.String()); hookErr != nil {
s.logger.Errorf("An error occurred calling an error hook: %s", hookErr)
}
}
s.logger.Fatalf("Fatal error running backup: %s", err) s.logger.Fatalf("Fatal error running backup: %s", err)
} }
} }
@ -526,3 +568,22 @@ func join(errs ...error) error {
} }
return errors.New("[" + strings.Join(msgs, ", ") + "]") return errors.New("[" + strings.Join(msgs, ", ") + "]")
} }
// buffer takes an io.Writer and returns a wrapped version of the
// writer that writes to both the original target as well as the returned buffer
func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {
buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w}
return buffering, &buffering.buf
}
type bufferingWriter struct {
buf bytes.Buffer
writer io.Writer
}
func (b *bufferingWriter) Write(p []byte) (n int, err error) {
if n, err := b.buf.Write(p); err != nil {
return n, fmt.Errorf("bufferingWriter: error writing to buffer: %w", err)
}
return b.writer.Write(p)
}

2
go.mod
View File

@ -20,6 +20,7 @@ require (
github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df // indirect
github.com/gogo/protobuf v1.3.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.0 // indirect github.com/golang/protobuf v1.5.0 // indirect
github.com/google/uuid v1.2.0 // indirect github.com/google/uuid v1.2.0 // indirect
@ -41,5 +42,6 @@ require (
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
google.golang.org/grpc v1.33.2 // indirect google.golang.org/grpc v1.33.2 // indirect
google.golang.org/protobuf v1.26.0 // indirect google.golang.org/protobuf v1.26.0 // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/ini.v1 v1.57.0 // indirect gopkg.in/ini.v1 v1.57.0 // indirect
) )

4
go.sum
View File

@ -254,6 +254,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df h1:Bao6dhmbTA1KFVxmJ6nBoMuOJit2yjEgLJpIMYpop0E=
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df/go.mod h1:GJr+FCSXshIwgHBtLglIg9M2l2kQSi6QjVAngtzI08Y=
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@ -908,6 +910,8 @@ google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/l
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=