Added custom notification messages using text/template (#60)

* Added custom notification messages using text/template

* Change notification template path and removed automatic newline trim

* Added stats and changed structure of template params

* Stat file hotfix

* Embedded and fixed default notification templates


Fix

* Changed Output to LogOutput

* Changed stats integer to unsigned

* Bytes formatting in template func


fix

* Changed Archives to Storages

* Removed unecessary sleep for pruning leeway

* Set EndTime after pruning is completed

* Added custom notifications documentation

* Added 5s sleep in swarm test

* Fixed documentation

* Dockerfile copies all files in cmd/backup

Dockerfile

@@ -6,7 +6,7 @@ FROM golang:1.17-alpine as builder
 WORKDIR /app
 COPY go.mod go.sum ./
 RUN go mod download
-COPY cmd/backup/main.go ./cmd/backup/main.go
+COPY cmd/backup ./cmd/backup/
 RUN go build -o backup cmd/backup/main.go
 
 FROM alpine:3.15

README.md

@@ -408,6 +408,30 @@ Refer to the documentation of [shoutrrr][shoutrrr-docs] to find out about option
 
 [shoutrrr-docs]: https://containrrr.dev/shoutrrr/v0.5/services/overview/
 
+### Customize notifications
+
+The title and body of the notifications can be easily tailored to your needs using [go templates](https://pkg.go.dev/text/template).
+Templates must be mounted inside the container in `/etc/dockervolumebackup/notifications.d/`: any file inside this directory will be parsed.
+
+The files have to define [nested templates](https://pkg.go.dev/text/template#hdr-Nested_template_definitions) in order to override the original values. An example:
+```
+{{ define "title_success" -}}
+✅ Successfully ran backup {{ .Config.BackupStopContainerLabel }}
+{{- end }}
+
+{{ define "body_success" -}}
+▶️ Start time: {{ .Stats.StartTime | formatTime }}
+⏹️ End time: {{ .Stats.EndTime | formatTime }}
+⌛ Took time: {{ .Stats.TookTime }}
+🛑 Stopped containers: {{ .Stats.Containers.Stopped }}/{{ .Stats.Containers.All }} ({{ .Stats.Containers.StopErrors }} errors)
+⚖️ Backup size: {{ .Stats.BackupFile.Size | formatBytesBin }} / {{ .Stats.BackupFile.Size | formatBytesDec }}
+🗑️ Pruned backups: {{ .Stats.Storages.Local.Pruned }}/{{ .Stats.Storages.Local.Total }} ({{ .Stats.Storages.Local.PruneErrors }} errors)
+{{- end }}
+```
+
+Overridable template names are: `title_success`, `body_success`, `title_failure`, `body_failure`.
+
+For a full list of available variables and functions, see [this page](https://github.com/offen/docker-volume-backup/blob/master/docs/NOTIFICATION-TEMPLATES.md).
 
 ### Encrypting your backup using GPG
 

cmd/backup/main.go

@@ -6,6 +6,7 @@ package main
 import (
 	"bytes"
 	"context"
+	_ "embed"
 	"errors"
 	"fmt"
 	"io"
@@ -15,6 +16,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/containrrr/shoutrrr"
@@ -36,6 +38,9 @@ import (
 	"golang.org/x/crypto/openpgp"
 )
 
+//go:embed notifications.tmpl
+var defaultNotifications string
+
 func main() {
 	unlock := lock("/var/lock/dockervolumebackup.lock")
 	defer unlock()
@@ -83,6 +88,8 @@ func main() {
 	s.must(s.encryptBackup())
 	s.must(s.copyBackup())
 	s.must(s.pruneOldBackups())
+	s.stats.EndTime = time.Now()
+	s.stats.TookTime = s.stats.EndTime.Sub(s.stats.EndTime)
 }
 
 // script holds all the stateful information required to orchestrate a
@@ -93,17 +100,64 @@ type script struct {
 	webdavClient *gowebdav.Client
 	logger       *logrus.Logger
 	sender       *router.ServiceRouter
+	template     *template.Template
 	hooks        []hook
 	hookLevel    hookLevel
 
-	start  time.Time
 	file  string
-	output *bytes.Buffer
+	stats *Stats
 
-	c *config
+	c *Config
 }
 
-type config struct {
+// ContainersStats stats about the docker containers
+type ContainersStats struct {
+	All        uint
+	ToStop     uint
+	Stopped    uint
+	StopErrors uint
+}
+
+// BackupFileStats stats about the created backup file
+type BackupFileStats struct {
+	Name     string
+	FullPath string
+	Size     uint64
+}
+
+// StorageStats stats about the status of an archival directory
+type StorageStats struct {
+	Total       uint
+	Pruned      uint
+	PruneErrors uint
+}
+
+// StoragesStats stats about each possible archival location (Local, WebDAV, S3)
+type StoragesStats struct {
+	Local  StorageStats
+	WebDAV StorageStats
+	S3     StorageStats
+}
+
+// Stats global stats regarding script execution
+type Stats struct {
+	StartTime  time.Time
+	EndTime    time.Time
+	TookTime   time.Duration
+	LogOutput  *bytes.Buffer
+	Containers ContainersStats
+	BackupFile BackupFileStats
+	Storages   StoragesStats
+}
+
+// NotificationData data to be passed to the notification templates
+type NotificationData struct {
+	Error  error
+	Config *Config
+	Stats  *Stats
+}
+
+type Config struct {
 	BackupSources              string        `split_words:"true" default:"/backup"`
 	BackupFilename             string        `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
 	BackupFilenameExpand       bool          `split_words:"true"`
@@ -146,15 +200,18 @@ var msgBackupFailed = "backup run failed"
 func newScript() (*script, error) {
 	stdOut, logBuffer := buffer(os.Stdout)
 	s := &script{
-		c: &config{},
+		c: &Config{},
 		logger: &logrus.Logger{
 			Out:       stdOut,
 			Formatter: new(logrus.TextFormatter),
 			Hooks:     make(logrus.LevelHooks),
 			Level:     logrus.InfoLevel,
 		},
-		start:  time.Now(),
+		stats: &Stats{
-		output: logBuffer,
+			StartTime: time.Now(),
+			LogOutput: logBuffer,
+			Storages:  StoragesStats{},
+		},
 	}
 
 	if err := envconfig.Process("", s.c); err != nil {
@@ -167,7 +224,7 @@ func newScript() (*script, error) {
 		s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
 		s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix)
 	}
-	s.file = timeutil.Strftime(&s.start, s.file)
+	s.file = timeutil.Strftime(&s.stats.StartTime, s.file)
 
 	_, err := os.Stat("/var/run/docker.sock")
 	if !os.IsNotExist(err) {
@@ -273,6 +330,31 @@ func newScript() (*script, error) {
 		})
 	}
 
+	tmpl := template.New("")
+	tmpl.Funcs(template.FuncMap{
+		"formatTime": func(t time.Time) string {
+			return t.Format(time.RFC3339)
+		},
+		"formatBytesDec": func(bytes uint64) string {
+			return formatBytes(bytes, true)
+		},
+		"formatBytesBin": func(bytes uint64) string {
+			return formatBytes(bytes, false)
+		},
+	})
+	tmpl, err = tmpl.Parse(defaultNotifications)
+	if err != nil {
+		return nil, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
+	}
+
+	if _, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil {
+		tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*")
+		if err != nil {
+			return nil, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
+		}
+	}
+	s.template = tmpl
+
 	return s, nil
 }
 
@@ -283,28 +365,39 @@ func (s *script) registerHook(level hookLevel, action func(err error) error) {
 	s.hooks = append(s.hooks, hook{level, action})
 }
 
-// notifyFailure sends a notification about a failed backup run
+// notify sends a notification using the given title and body templates.
-func (s *script) notifyFailure(err error) error {
+// Automatically creates notification data, adding the given error
-	body := fmt.Sprintf(
+func (s *script) notify(titleTemplate string, bodyTemplate string, err error) error {
-		"Running docker-volume-backup failed with error: %s\n\nLog output of the failed run was:\n\n%s\n", err, s.output.String(),
+	params := NotificationData{
-	)
+		Error:  err,
-	title := fmt.Sprintf("Failure running docker-volume-backup at %s", s.start.Format(time.RFC3339))
+		Stats:  s.stats,
-	if err := s.sendNotification(title, body); err != nil {
+		Config: s.c,
+	}
+
+	titleBuf := &bytes.Buffer{}
+	if err := s.template.ExecuteTemplate(titleBuf, titleTemplate, params); err != nil {
+		return fmt.Errorf("notifyFailure: error executing %s template: %w", titleTemplate, err)
+	}
+
+	bodyBuf := &bytes.Buffer{}
+	if err := s.template.ExecuteTemplate(bodyBuf, bodyTemplate, params); err != nil {
+		return fmt.Errorf("notifyFailure: error executing %s template: %w", bodyTemplate, err)
+	}
+
+	if err := s.sendNotification(titleBuf.String(), bodyBuf.String()); err != nil {
 		return fmt.Errorf("notifyFailure: error notifying: %w", err)
 	}
 	return nil
 }
 
+// notifyFailure sends a notification about a failed backup run
+func (s *script) notifyFailure(err error) error {
+	return s.notify("title_failure", "body_failure", err)
+}
+
 // notifyFailure sends a notification about a successful backup run
 func (s *script) notifySuccess() error {
-	title := fmt.Sprintf("Success running docker-volume-backup at %s", s.start.Format(time.RFC3339))
+	return s.notify("title_success", "body_success", nil)
-	body := fmt.Sprintf(
-		"Running docker-volume-backup succeeded.\n\nLog output was:\n\n%s\n", s.output.String(),
-	)
-	if err := s.sendNotification(title, body); err != nil {
-		return fmt.Errorf("notifySuccess: error notifying: %w", err)
-	}
-	return nil
 }
 
 // sendNotification sends a notification to all configured third party services
@@ -382,6 +475,12 @@ func (s *script) stopContainers() (func() error, error) {
 		)
 	}
 
+	s.stats.Containers = ContainersStats{
+		All:     uint(len(allContainers)),
+		ToStop:  uint(len(containersToStop)),
+		Stopped: uint(len(stoppedContainers)),
+	}
+
 	return func() error {
 		servicesRequiringUpdate := map[string]struct{}{}
 
@@ -525,6 +624,17 @@ func (s *script) encryptBackup() error {
 // as per the given configuration.
 func (s *script) copyBackup() error {
 	_, name := path.Split(s.file)
+	if stat, err := os.Stat(s.file); err != nil {
+		return fmt.Errorf("copyBackup: unable to stat backup file: %w", err)
+	} else {
+		size := stat.Size()
+		s.stats.BackupFile = BackupFileStats{
+			Size:     uint64(size),
+			Name:     name,
+			FullPath: s.file,
+		}
+	}
+
 	if s.minioClient != nil {
 		if _, err := s.minioClient.FPutObject(context.Background(), s.c.AwsS3BucketName, filepath.Join(s.c.AwsS3Path, name), s.file, minio.PutObjectOptions{
 			ContentType: "application/tar+gzip",
@@ -575,12 +685,7 @@ func (s *script) pruneOldBackups() error {
 		return nil
 	}
 
-	if s.c.BackupPruningLeeway != 0 {
+	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
-		s.logger.Infof("Sleeping for %s before pruning backups.", s.c.BackupPruningLeeway)
-		time.Sleep(s.c.BackupPruningLeeway)
-	}
-
-	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays))
 
 	// Prune minio/S3 backups
 	if s.minioClient != nil {
@@ -604,6 +709,10 @@ func (s *script) pruneOldBackups() error {
 			}
 		}
 
+		s.stats.Storages.S3 = StorageStats{
+			Total:  uint(lenCandidates),
+			Pruned: uint(len(matches)),
+		}
 		if len(matches) != 0 && len(matches) != lenCandidates {
 			objectsCh := make(chan minio.ObjectInfo)
 			go func() {
@@ -619,6 +728,7 @@ func (s *script) pruneOldBackups() error {
 					removeErrors = append(removeErrors, result.Err)
 				}
 			}
+			s.stats.Storages.S3.PruneErrors = uint(len(removeErrors))
 
 			if len(removeErrors) != 0 {
 				return fmt.Errorf(
@@ -627,10 +737,11 @@ func (s *script) pruneOldBackups() error {
 					join(removeErrors...),
 				)
 			}
+
 			s.logger.Infof(
 				"Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.",
-				len(matches),
+				s.stats.Storages.S3.Pruned,
-				lenCandidates,
+				s.stats.Storages.S3.Total,
 				s.c.BackupRetentionDays,
 			)
 		} else if len(matches) != 0 && len(matches) == lenCandidates {
@@ -659,14 +770,33 @@ func (s *script) pruneOldBackups() error {
 			}
 		}
 
+		s.stats.Storages.WebDAV = StorageStats{
+			Total:  uint(lenCandidates),
+			Pruned: uint(len(matches)),
+		}
 		if len(matches) != 0 && len(matches) != lenCandidates {
+			var removeErrors []error
 			for _, match := range matches {
 				if err := s.webdavClient.Remove(filepath.Join(s.c.WebdavPath, match.Name())); err != nil {
-					return fmt.Errorf("pruneOldBackups: error removing a file from remote storage: %w", err)
+					removeErrors = append(removeErrors, err)
-				}
+				} else {
 					s.logger.Infof("Pruned %s from WebDAV: %s", match.Name(), filepath.Join(s.c.WebdavUrl, s.c.WebdavPath))
 				}
-			s.logger.Infof("Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.", len(matches), lenCandidates, s.c.BackupRetentionDays)
+			}
+			s.stats.Storages.WebDAV.PruneErrors = uint(len(removeErrors))
+			if len(removeErrors) != 0 {
+				return fmt.Errorf(
+					"pruneOldBackups: %d error(s) removing files from remote storage: %w",
+					len(removeErrors),
+					join(removeErrors...),
+				)
+			}
+			s.logger.Infof(
+				"Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.",
+				s.stats.Storages.WebDAV.Pruned,
+				s.stats.Storages.WebDAV.Total,
+				s.c.BackupRetentionDays,
+			)
 		} else if len(matches) != 0 && len(matches) == lenCandidates {
 			s.logger.Warnf("The current configuration would delete all %d remote backup copies.", len(matches))
 			s.logger.Warn("Refusing to do so, please check your configuration.")
@@ -721,6 +851,10 @@ func (s *script) pruneOldBackups() error {
 			}
 		}
 
+		s.stats.Storages.Local = StorageStats{
+			Total:  uint(len(candidates)),
+			Pruned: uint(len(matches)),
+		}
 		if len(matches) != 0 && len(matches) != len(candidates) {
 			var removeErrors []error
 			for _, match := range matches {
@@ -729,6 +863,7 @@ func (s *script) pruneOldBackups() error {
 				}
 			}
 			if len(removeErrors) != 0 {
+				s.stats.Storages.Local.PruneErrors = uint(len(removeErrors))
 				return fmt.Errorf(
 					"pruneOldBackups: %d error(s) deleting local files, starting with: %w",
 					len(removeErrors),
@@ -737,8 +872,8 @@ func (s *script) pruneOldBackups() error {
 			}
 			s.logger.Infof(
 				"Pruned %d out of %d local backup(s) as their age exceeded the configured retention period of %d days.",
-				len(matches),
+				s.stats.Storages.Local.Pruned,
-				len(candidates),
+				s.stats.Storages.Local.Total,
 				s.c.BackupRetentionDays,
 			)
 		} else if len(matches) != 0 && len(matches) == len(candidates) {
@@ -856,6 +991,26 @@ func join(errs ...error) error {
 	return errors.New("[" + strings.Join(msgs, ", ") + "]")
 }
 
+// formatBytes converts an amount of bytes in a human-readable representation
+// the decimal parameter specifies if using powers of 1000 (decimal) or powers of 1024 (binary)
+func formatBytes(b uint64, decimal bool) string {
+	unit := uint64(1024)
+	format := "%.1f %ciB"
+	if decimal {
+		unit = uint64(1000)
+		format = "%.1f %cB"
+	}
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := unit, 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf(format, float64(b)/float64(div), "kMGTPE"[exp])
+}
+
 // buffer takes an io.Writer and returns a wrapped version of the
 // writer that writes to both the original target as well as the returned buffer
 func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {

cmd/backup/notifications.tmpl

@@ -0,0 +1,26 @@
+{{ define "title_failure" -}}
+Failure running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
+{{- end }}
+
+
+{{ define "body_failure" -}}
+Running docker-volume-backup failed with error: {{ .Error }}
+
+Log output of the failed run was:
+
+{{ .Stats.LogOutput }}
+{{- end }}
+
+
+{{ define "title_success" -}}
+Success running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
+{{- end }}
+
+
+{{ define "body_success" -}}
+Running docker-volume-backup succeeded.
+
+Log output was:
+
+{{ .Stats.LogOutput }}
+{{- end }}

docs/NOTIFICATION-TEMPLATES.md

@@ -0,0 +1,38 @@
+# Notification templates reference
+
+In order to customize title and body of notifications you'll have to write a [go template](https://pkg.go.dev/text/template) and mount it inside the `/etc/dockervolumebackup/notifications.d/` directory.
+
+Configuration, data about the backup run and helper functions will be passed to this template, this page documents them fully.
+
+## Data
+Here is a list of all data passed to the template:
+
+* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`)
+* `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates
+* `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available.
+  * `StartTime`: time when the script started execution
+  * `EndTime`: time when the backup has completed successfully (after pruning)
+  * `TookTime`: amount of time it took for the backup to run. (equal to `EndTime - StartTime`)
+  * `LogOutput`: full log of the application
+  * `Containers`: object containing stats about the docker containers
+    * `All`: total number of containers
+    * `ToStop`: number of containers matched by the stop rule
+    * `Stopped`: number of containers successfully stopped
+    * `StopErrors`: number of containers that were unable to be stopped (equal to `ToStop - Stopped`)
+  * `BackupFile`: object containing information about the backup file
+    * `Name`: name of the backup file (e.g. `backup-2022-02-11T01-00-00.tar.gz`)
+    * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
+    * `Size`: size in bytes of the backup file
+  * `Storages`: object that holds stats about each storage
+    * `Local`, `S3` or `WebDAV`:
+      * `Total`: total number of backup files
+      * `Pruned`: number of backup files that were deleted due to pruning rule
+      * `PruneErrors`: number of backup files that were unable to be pruned
+
+## Functions
+
+Some formatting functions are also available:
+
+* `formatTime`: formats a time object using [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format (e.g. `2022-02-11T01:00:00Z`)
+* `formatBytesBin`: formats an amount of bytes using powers of 1024 (e.g. `7055258` bytes will be `6.7 MiB`) 
+* `formatBytesDec`: formats an amount of bytes using powers of 1000 (e.g. `7055258` bytes will be `7.1 MB`)

test/swarm/run.sh

@@ -23,6 +23,7 @@ docker run --rm -it \
 
 echo "[TEST:PASS] Found relevant files in untared backup."
 
+sleep 5
 if [ "$(docker ps -q | wc -l)" != "5" ]; then
   echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
   docker ps -a