From 8dfdd145273fbcf227ec47071f75168a6e286b14 Mon Sep 17 00:00:00 2001 From: Mauro Molin Date: Fri, 11 Feb 2022 20:05:16 +0100 Subject: [PATCH] Added custom notification messages using text/template (#60) * Added custom notification messages using text/template * Change notification template path and removed automatic newline trim * Added stats and changed structure of template params * Stat file hotfix * Embedded and fixed default notification templates Fix * Changed Output to LogOutput * Changed stats integer to unsigned * Bytes formatting in template func fix * Changed Archives to Storages * Removed unecessary sleep for pruning leeway * Set EndTime after pruning is completed * Added custom notifications documentation * Added 5s sleep in swarm test * Fixed documentation * Dockerfile copies all files in cmd/backup --- Dockerfile | 2 +- README.md | 24 ++++ cmd/backup/main.go | 229 +++++++++++++++++++++++++++------ cmd/backup/notifications.tmpl | 26 ++++ docs/NOTIFICATION-TEMPLATES.md | 38 ++++++ test/swarm/run.sh | 1 + 6 files changed, 282 insertions(+), 38 deletions(-) create mode 100644 cmd/backup/notifications.tmpl create mode 100644 docs/NOTIFICATION-TEMPLATES.md diff --git a/Dockerfile b/Dockerfile index 35a3b07..7fd3b99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ FROM golang:1.17-alpine as builder WORKDIR /app COPY go.mod go.sum ./ RUN go mod download -COPY cmd/backup/main.go ./cmd/backup/main.go +COPY cmd/backup ./cmd/backup/ RUN go build -o backup cmd/backup/main.go FROM alpine:3.15 diff --git a/README.md b/README.md index 120b323..3065b21 100644 --- a/README.md +++ b/README.md @@ -408,6 +408,30 @@ Refer to the documentation of [shoutrrr][shoutrrr-docs] to find out about option [shoutrrr-docs]: https://containrrr.dev/shoutrrr/v0.5/services/overview/ +### Customize notifications + +The title and body of the notifications can be easily tailored to your needs using [go templates](https://pkg.go.dev/text/template). +Templates must be mounted inside the container in `/etc/dockervolumebackup/notifications.d/`: any file inside this directory will be parsed. + +The files have to define [nested templates](https://pkg.go.dev/text/template#hdr-Nested_template_definitions) in order to override the original values. An example: +``` +{{ define "title_success" -}} +✅ Successfully ran backup {{ .Config.BackupStopContainerLabel }} +{{- end }} + +{{ define "body_success" -}} +▶️ Start time: {{ .Stats.StartTime | formatTime }} +⏹️ End time: {{ .Stats.EndTime | formatTime }} +⌛ Took time: {{ .Stats.TookTime }} +🛑 Stopped containers: {{ .Stats.Containers.Stopped }}/{{ .Stats.Containers.All }} ({{ .Stats.Containers.StopErrors }} errors) +⚖️ Backup size: {{ .Stats.BackupFile.Size | formatBytesBin }} / {{ .Stats.BackupFile.Size | formatBytesDec }} +🗑️ Pruned backups: {{ .Stats.Storages.Local.Pruned }}/{{ .Stats.Storages.Local.Total }} ({{ .Stats.Storages.Local.PruneErrors }} errors) +{{- end }} +``` + +Overridable template names are: `title_success`, `body_success`, `title_failure`, `body_failure`. + +For a full list of available variables and functions, see [this page](https://github.com/offen/docker-volume-backup/blob/master/docs/NOTIFICATION-TEMPLATES.md). ### Encrypting your backup using GPG diff --git a/cmd/backup/main.go b/cmd/backup/main.go index fd67f1e..d1c2b21 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -6,6 +6,7 @@ package main import ( "bytes" "context" + _ "embed" "errors" "fmt" "io" @@ -15,6 +16,7 @@ import ( "path/filepath" "sort" "strings" + "text/template" "time" "github.com/containrrr/shoutrrr" @@ -36,6 +38,9 @@ import ( "golang.org/x/crypto/openpgp" ) +//go:embed notifications.tmpl +var defaultNotifications string + func main() { unlock := lock("/var/lock/dockervolumebackup.lock") defer unlock() @@ -83,6 +88,8 @@ func main() { s.must(s.encryptBackup()) s.must(s.copyBackup()) s.must(s.pruneOldBackups()) + s.stats.EndTime = time.Now() + s.stats.TookTime = s.stats.EndTime.Sub(s.stats.EndTime) } // script holds all the stateful information required to orchestrate a @@ -93,17 +100,64 @@ type script struct { webdavClient *gowebdav.Client logger *logrus.Logger sender *router.ServiceRouter + template *template.Template hooks []hook hookLevel hookLevel - start time.Time - file string - output *bytes.Buffer + file string + stats *Stats - c *config + c *Config } -type config struct { +// ContainersStats stats about the docker containers +type ContainersStats struct { + All uint + ToStop uint + Stopped uint + StopErrors uint +} + +// BackupFileStats stats about the created backup file +type BackupFileStats struct { + Name string + FullPath string + Size uint64 +} + +// StorageStats stats about the status of an archival directory +type StorageStats struct { + Total uint + Pruned uint + PruneErrors uint +} + +// StoragesStats stats about each possible archival location (Local, WebDAV, S3) +type StoragesStats struct { + Local StorageStats + WebDAV StorageStats + S3 StorageStats +} + +// Stats global stats regarding script execution +type Stats struct { + StartTime time.Time + EndTime time.Time + TookTime time.Duration + LogOutput *bytes.Buffer + Containers ContainersStats + BackupFile BackupFileStats + Storages StoragesStats +} + +// NotificationData data to be passed to the notification templates +type NotificationData struct { + Error error + Config *Config + Stats *Stats +} + +type Config struct { BackupSources string `split_words:"true" default:"/backup"` BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` BackupFilenameExpand bool `split_words:"true"` @@ -146,15 +200,18 @@ var msgBackupFailed = "backup run failed" func newScript() (*script, error) { stdOut, logBuffer := buffer(os.Stdout) s := &script{ - c: &config{}, + c: &Config{}, logger: &logrus.Logger{ Out: stdOut, Formatter: new(logrus.TextFormatter), Hooks: make(logrus.LevelHooks), Level: logrus.InfoLevel, }, - start: time.Now(), - output: logBuffer, + stats: &Stats{ + StartTime: time.Now(), + LogOutput: logBuffer, + Storages: StoragesStats{}, + }, } if err := envconfig.Process("", s.c); err != nil { @@ -167,7 +224,7 @@ func newScript() (*script, error) { s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink) s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix) } - s.file = timeutil.Strftime(&s.start, s.file) + s.file = timeutil.Strftime(&s.stats.StartTime, s.file) _, err := os.Stat("/var/run/docker.sock") if !os.IsNotExist(err) { @@ -273,6 +330,31 @@ func newScript() (*script, error) { }) } + tmpl := template.New("") + tmpl.Funcs(template.FuncMap{ + "formatTime": func(t time.Time) string { + return t.Format(time.RFC3339) + }, + "formatBytesDec": func(bytes uint64) string { + return formatBytes(bytes, true) + }, + "formatBytesBin": func(bytes uint64) string { + return formatBytes(bytes, false) + }, + }) + tmpl, err = tmpl.Parse(defaultNotifications) + if err != nil { + return nil, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err) + } + + if _, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil { + tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*") + if err != nil { + return nil, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err) + } + } + s.template = tmpl + return s, nil } @@ -283,28 +365,39 @@ func (s *script) registerHook(level hookLevel, action func(err error) error) { s.hooks = append(s.hooks, hook{level, action}) } -// notifyFailure sends a notification about a failed backup run -func (s *script) notifyFailure(err error) error { - body := fmt.Sprintf( - "Running docker-volume-backup failed with error: %s\n\nLog output of the failed run was:\n\n%s\n", err, s.output.String(), - ) - title := fmt.Sprintf("Failure running docker-volume-backup at %s", s.start.Format(time.RFC3339)) - if err := s.sendNotification(title, body); err != nil { +// notify sends a notification using the given title and body templates. +// Automatically creates notification data, adding the given error +func (s *script) notify(titleTemplate string, bodyTemplate string, err error) error { + params := NotificationData{ + Error: err, + Stats: s.stats, + Config: s.c, + } + + titleBuf := &bytes.Buffer{} + if err := s.template.ExecuteTemplate(titleBuf, titleTemplate, params); err != nil { + return fmt.Errorf("notifyFailure: error executing %s template: %w", titleTemplate, err) + } + + bodyBuf := &bytes.Buffer{} + if err := s.template.ExecuteTemplate(bodyBuf, bodyTemplate, params); err != nil { + return fmt.Errorf("notifyFailure: error executing %s template: %w", bodyTemplate, err) + } + + if err := s.sendNotification(titleBuf.String(), bodyBuf.String()); err != nil { return fmt.Errorf("notifyFailure: error notifying: %w", err) } return nil } +// notifyFailure sends a notification about a failed backup run +func (s *script) notifyFailure(err error) error { + return s.notify("title_failure", "body_failure", err) +} + // notifyFailure sends a notification about a successful backup run func (s *script) notifySuccess() error { - title := fmt.Sprintf("Success running docker-volume-backup at %s", s.start.Format(time.RFC3339)) - body := fmt.Sprintf( - "Running docker-volume-backup succeeded.\n\nLog output was:\n\n%s\n", s.output.String(), - ) - if err := s.sendNotification(title, body); err != nil { - return fmt.Errorf("notifySuccess: error notifying: %w", err) - } - return nil + return s.notify("title_success", "body_success", nil) } // sendNotification sends a notification to all configured third party services @@ -382,6 +475,12 @@ func (s *script) stopContainers() (func() error, error) { ) } + s.stats.Containers = ContainersStats{ + All: uint(len(allContainers)), + ToStop: uint(len(containersToStop)), + Stopped: uint(len(stoppedContainers)), + } + return func() error { servicesRequiringUpdate := map[string]struct{}{} @@ -525,6 +624,17 @@ func (s *script) encryptBackup() error { // as per the given configuration. func (s *script) copyBackup() error { _, name := path.Split(s.file) + if stat, err := os.Stat(s.file); err != nil { + return fmt.Errorf("copyBackup: unable to stat backup file: %w", err) + } else { + size := stat.Size() + s.stats.BackupFile = BackupFileStats{ + Size: uint64(size), + Name: name, + FullPath: s.file, + } + } + if s.minioClient != nil { if _, err := s.minioClient.FPutObject(context.Background(), s.c.AwsS3BucketName, filepath.Join(s.c.AwsS3Path, name), s.file, minio.PutObjectOptions{ ContentType: "application/tar+gzip", @@ -575,12 +685,7 @@ func (s *script) pruneOldBackups() error { return nil } - if s.c.BackupPruningLeeway != 0 { - s.logger.Infof("Sleeping for %s before pruning backups.", s.c.BackupPruningLeeway) - time.Sleep(s.c.BackupPruningLeeway) - } - - deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)) + deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway) // Prune minio/S3 backups if s.minioClient != nil { @@ -604,6 +709,10 @@ func (s *script) pruneOldBackups() error { } } + s.stats.Storages.S3 = StorageStats{ + Total: uint(lenCandidates), + Pruned: uint(len(matches)), + } if len(matches) != 0 && len(matches) != lenCandidates { objectsCh := make(chan minio.ObjectInfo) go func() { @@ -619,6 +728,7 @@ func (s *script) pruneOldBackups() error { removeErrors = append(removeErrors, result.Err) } } + s.stats.Storages.S3.PruneErrors = uint(len(removeErrors)) if len(removeErrors) != 0 { return fmt.Errorf( @@ -627,10 +737,11 @@ func (s *script) pruneOldBackups() error { join(removeErrors...), ) } + s.logger.Infof( "Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.", - len(matches), - lenCandidates, + s.stats.Storages.S3.Pruned, + s.stats.Storages.S3.Total, s.c.BackupRetentionDays, ) } else if len(matches) != 0 && len(matches) == lenCandidates { @@ -659,14 +770,33 @@ func (s *script) pruneOldBackups() error { } } + s.stats.Storages.WebDAV = StorageStats{ + Total: uint(lenCandidates), + Pruned: uint(len(matches)), + } if len(matches) != 0 && len(matches) != lenCandidates { + var removeErrors []error for _, match := range matches { if err := s.webdavClient.Remove(filepath.Join(s.c.WebdavPath, match.Name())); err != nil { - return fmt.Errorf("pruneOldBackups: error removing a file from remote storage: %w", err) + removeErrors = append(removeErrors, err) + } else { + s.logger.Infof("Pruned %s from WebDAV: %s", match.Name(), filepath.Join(s.c.WebdavUrl, s.c.WebdavPath)) } - s.logger.Infof("Pruned %s from WebDAV: %s", match.Name(), filepath.Join(s.c.WebdavUrl, s.c.WebdavPath)) } - s.logger.Infof("Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.", len(matches), lenCandidates, s.c.BackupRetentionDays) + s.stats.Storages.WebDAV.PruneErrors = uint(len(removeErrors)) + if len(removeErrors) != 0 { + return fmt.Errorf( + "pruneOldBackups: %d error(s) removing files from remote storage: %w", + len(removeErrors), + join(removeErrors...), + ) + } + s.logger.Infof( + "Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.", + s.stats.Storages.WebDAV.Pruned, + s.stats.Storages.WebDAV.Total, + s.c.BackupRetentionDays, + ) } else if len(matches) != 0 && len(matches) == lenCandidates { s.logger.Warnf("The current configuration would delete all %d remote backup copies.", len(matches)) s.logger.Warn("Refusing to do so, please check your configuration.") @@ -721,6 +851,10 @@ func (s *script) pruneOldBackups() error { } } + s.stats.Storages.Local = StorageStats{ + Total: uint(len(candidates)), + Pruned: uint(len(matches)), + } if len(matches) != 0 && len(matches) != len(candidates) { var removeErrors []error for _, match := range matches { @@ -729,6 +863,7 @@ func (s *script) pruneOldBackups() error { } } if len(removeErrors) != 0 { + s.stats.Storages.Local.PruneErrors = uint(len(removeErrors)) return fmt.Errorf( "pruneOldBackups: %d error(s) deleting local files, starting with: %w", len(removeErrors), @@ -737,8 +872,8 @@ func (s *script) pruneOldBackups() error { } s.logger.Infof( "Pruned %d out of %d local backup(s) as their age exceeded the configured retention period of %d days.", - len(matches), - len(candidates), + s.stats.Storages.Local.Pruned, + s.stats.Storages.Local.Total, s.c.BackupRetentionDays, ) } else if len(matches) != 0 && len(matches) == len(candidates) { @@ -856,6 +991,26 @@ func join(errs ...error) error { return errors.New("[" + strings.Join(msgs, ", ") + "]") } +// formatBytes converts an amount of bytes in a human-readable representation +// the decimal parameter specifies if using powers of 1000 (decimal) or powers of 1024 (binary) +func formatBytes(b uint64, decimal bool) string { + unit := uint64(1024) + format := "%.1f %ciB" + if decimal { + unit = uint64(1000) + format = "%.1f %cB" + } + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := unit, 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf(format, float64(b)/float64(div), "kMGTPE"[exp]) +} + // buffer takes an io.Writer and returns a wrapped version of the // writer that writes to both the original target as well as the returned buffer func buffer(w io.Writer) (io.Writer, *bytes.Buffer) { diff --git a/cmd/backup/notifications.tmpl b/cmd/backup/notifications.tmpl new file mode 100644 index 0000000..3a51a03 --- /dev/null +++ b/cmd/backup/notifications.tmpl @@ -0,0 +1,26 @@ +{{ define "title_failure" -}} +Failure running docker-volume-backup at {{ .Stats.StartTime | formatTime }} +{{- end }} + + +{{ define "body_failure" -}} +Running docker-volume-backup failed with error: {{ .Error }} + +Log output of the failed run was: + +{{ .Stats.LogOutput }} +{{- end }} + + +{{ define "title_success" -}} +Success running docker-volume-backup at {{ .Stats.StartTime | formatTime }} +{{- end }} + + +{{ define "body_success" -}} +Running docker-volume-backup succeeded. + +Log output was: + +{{ .Stats.LogOutput }} +{{- end }} diff --git a/docs/NOTIFICATION-TEMPLATES.md b/docs/NOTIFICATION-TEMPLATES.md new file mode 100644 index 0000000..a817d56 --- /dev/null +++ b/docs/NOTIFICATION-TEMPLATES.md @@ -0,0 +1,38 @@ +# Notification templates reference + +In order to customize title and body of notifications you'll have to write a [go template](https://pkg.go.dev/text/template) and mount it inside the `/etc/dockervolumebackup/notifications.d/` directory. + +Configuration, data about the backup run and helper functions will be passed to this template, this page documents them fully. + +## Data +Here is a list of all data passed to the template: + +* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`) +* `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates +* `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available. + * `StartTime`: time when the script started execution + * `EndTime`: time when the backup has completed successfully (after pruning) + * `TookTime`: amount of time it took for the backup to run. (equal to `EndTime - StartTime`) + * `LogOutput`: full log of the application + * `Containers`: object containing stats about the docker containers + * `All`: total number of containers + * `ToStop`: number of containers matched by the stop rule + * `Stopped`: number of containers successfully stopped + * `StopErrors`: number of containers that were unable to be stopped (equal to `ToStop - Stopped`) + * `BackupFile`: object containing information about the backup file + * `Name`: name of the backup file (e.g. `backup-2022-02-11T01-00-00.tar.gz`) + * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`) + * `Size`: size in bytes of the backup file + * `Storages`: object that holds stats about each storage + * `Local`, `S3` or `WebDAV`: + * `Total`: total number of backup files + * `Pruned`: number of backup files that were deleted due to pruning rule + * `PruneErrors`: number of backup files that were unable to be pruned + +## Functions + +Some formatting functions are also available: + +* `formatTime`: formats a time object using [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format (e.g. `2022-02-11T01:00:00Z`) +* `formatBytesBin`: formats an amount of bytes using powers of 1024 (e.g. `7055258` bytes will be `6.7 MiB`) +* `formatBytesDec`: formats an amount of bytes using powers of 1000 (e.g. `7055258` bytes will be `7.1 MB`) diff --git a/test/swarm/run.sh b/test/swarm/run.sh index d8cfdf0..646b9e5 100755 --- a/test/swarm/run.sh +++ b/test/swarm/run.sh @@ -23,6 +23,7 @@ docker run --rm -it \ echo "[TEST:PASS] Found relevant files in untared backup." +sleep 5 if [ "$(docker ps -q | wc -l)" != "5" ]; then echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:" docker ps -a