offen/docker-volume-backup
2
0
Fork 0
mirror of https://github.com/offen/docker-volume-backup.git synced 2024-11-21 21:10:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Added custom notification messages using text/template (#60)

Browse Source 
* Added custom notification messages using text/template

* Change notification template path and removed automatic newline trim

* Added stats and changed structure of template params

* Stat file hotfix

* Embedded and fixed default notification templates


Fix

* Changed Output to LogOutput

* Changed stats integer to unsigned

* Bytes formatting in template func


fix

* Changed Archives to Storages

* Removed unecessary sleep for pruning leeway

* Set EndTime after pruning is completed

* Added custom notifications documentation

* Added 5s sleep in swarm test

* Fixed documentation

* Dockerfile copies all files in cmd/backup
This commit is contained in:
Mauro Molin 2022-02-11 20:05:16 +01:00 committed by GitHub
parent 3bb99a7117
commit 8dfdd14527
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 282 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -6,7 +6,7 @@ FROM golang:1.17-alpine as builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY cmd/backup/main.go ./cmd/backup/main.go
COPY cmd/backup ./cmd/backup/
RUN go build -o backup cmd/backup/main.go
FROM alpine:3.15

24
README.md
View File

@ -408,6 +408,30 @@ Refer to the documentation of [shoutrrr][shoutrrr-docs] to find out about option
[shoutrrr-docs]: https://containrrr.dev/shoutrrr/v0.5/services/overview/
### Customize notifications
The title and body of the notifications can be easily tailored to your needs using [go templates](https://pkg.go.dev/text/template).
Templates must be mounted inside the container in `/etc/dockervolumebackup/notifications.d/`: any file inside this directory will be parsed.
The files have to define [nested templates](https://pkg.go.dev/text/template#hdr-Nested_template_definitions) in order to override the original values. An example:
```
{{ define "title_success" -}}
✅ Successfully ran backup {{ .Config.BackupStopContainerLabel }}
{{- end }}
{{ define "body_success" -}}
▶️ Start time: {{ .Stats.StartTime | formatTime }}
⏹️ End time: {{ .Stats.EndTime | formatTime }}
⌛ Took time: {{ .Stats.TookTime }}
🛑 Stopped containers: {{ .Stats.Containers.Stopped }}/{{ .Stats.Containers.All }} ({{ .Stats.Containers.StopErrors }} errors)
⚖️ Backup size: {{ .Stats.BackupFile.Size | formatBytesBin }} / {{ .Stats.BackupFile.Size | formatBytesDec }}
🗑️ Pruned backups: {{ .Stats.Storages.Local.Pruned }}/{{ .Stats.Storages.Local.Total }} ({{ .Stats.Storages.Local.PruneErrors }} errors)
{{- end }}
```
Overridable template names are: `title_success`, `body_success`, `title_failure`, `body_failure`.
For a full list of available variables and functions, see [this page](https://github.com/offen/docker-volume-backup/blob/master/docs/NOTIFICATION-TEMPLATES.md).
### Encrypting your backup using GPG

227
cmd/backup/main.go
View File

@ -6,6 +6,7 @@ package main
import (
"bytes"
"context"
_ "embed"
"errors"
"fmt"
"io"
@ -15,6 +16,7 @@ import (
"path/filepath"
"sort"
"strings"
"text/template"
"time"
"github.com/containrrr/shoutrrr"
@ -36,6 +38,9 @@ import (
"golang.org/x/crypto/openpgp"
)
//go:embed notifications.tmpl
var defaultNotifications string
func main() {
unlock := lock("/var/lock/dockervolumebackup.lock")
defer unlock()
@ -83,6 +88,8 @@ func main() {
s.must(s.encryptBackup())
s.must(s.copyBackup())
s.must(s.pruneOldBackups())
s.stats.EndTime = time.Now()
s.stats.TookTime = s.stats.EndTime.Sub(s.stats.EndTime)
}
// script holds all the stateful information required to orchestrate a
@ -93,17 +100,64 @@ type script struct {
webdavClient *gowebdav.Client
logger *logrus.Logger
sender *router.ServiceRouter
template *template.Template
hooks []hook
hookLevel hookLevel
start time.Time
file string
output *bytes.Buffer
stats *Stats
c *config
c *Config
}
type config struct {
// ContainersStats stats about the docker containers
type ContainersStats struct {
All uint
ToStop uint
Stopped uint
StopErrors uint
}
// BackupFileStats stats about the created backup file
type BackupFileStats struct {
Name string
FullPath string
Size uint64
}
// StorageStats stats about the status of an archival directory
type StorageStats struct {
Total uint
Pruned uint
PruneErrors uint
}
// StoragesStats stats about each possible archival location (Local, WebDAV, S3)
type StoragesStats struct {
Local StorageStats
WebDAV StorageStats
S3 StorageStats
}
// Stats global stats regarding script execution
type Stats struct {
StartTime time.Time
EndTime time.Time
TookTime time.Duration
LogOutput *bytes.Buffer
Containers ContainersStats
BackupFile BackupFileStats
Storages StoragesStats
}
// NotificationData data to be passed to the notification templates
type NotificationData struct {
Error error
Config *Config
Stats *Stats
}
type Config struct {
BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"`
BackupFilenameExpand bool `split_words:"true"`
@ -146,15 +200,18 @@ var msgBackupFailed = "backup run failed"
func newScript() (*script, error) {
stdOut, logBuffer := buffer(os.Stdout)
s := &script{
c: &config{},
c: &Config{},
logger: &logrus.Logger{
Out: stdOut,
Formatter: new(logrus.TextFormatter),
Hooks: make(logrus.LevelHooks),
Level: logrus.InfoLevel,
},
start: time.Now(),
output: logBuffer,
stats: &Stats{
StartTime: time.Now(),
LogOutput: logBuffer,
Storages: StoragesStats{},
},
}
if err := envconfig.Process("", s.c); err != nil {
@ -167,7 +224,7 @@ func newScript() (*script, error) {
s.c.BackupLatestSymlink = os.ExpandEnv(s.c.BackupLatestSymlink)
s.c.BackupPruningPrefix = os.ExpandEnv(s.c.BackupPruningPrefix)
}
s.file = timeutil.Strftime(&s.start, s.file)
s.file = timeutil.Strftime(&s.stats.StartTime, s.file)
_, err := os.Stat("/var/run/docker.sock")
if !os.IsNotExist(err) {
@ -273,6 +330,31 @@ func newScript() (*script, error) {
})
}
tmpl := template.New("")
tmpl.Funcs(template.FuncMap{
"formatTime": func(t time.Time) string {
return t.Format(time.RFC3339)
},
"formatBytesDec": func(bytes uint64) string {
return formatBytes(bytes, true)
},
"formatBytesBin": func(bytes uint64) string {
return formatBytes(bytes, false)
},
})
tmpl, err = tmpl.Parse(defaultNotifications)
if err != nil {
return nil, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
}
if _, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil {
tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*")
if err != nil {
return nil, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
}
}
s.template = tmpl
return s, nil
}
@ -283,28 +365,39 @@ func (s *script) registerHook(level hookLevel, action func(err error) error) {
s.hooks = append(s.hooks, hook{level, action})
}
// notifyFailure sends a notification about a failed backup run
func (s *script) notifyFailure(err error) error {
body := fmt.Sprintf(
"Running docker-volume-backup failed with error: %s\n\nLog output of the failed run was:\n\n%s\n", err, s.output.String(),
)
title := fmt.Sprintf("Failure running docker-volume-backup at %s", s.start.Format(time.RFC3339))
if err := s.sendNotification(title, body); err != nil {
// notify sends a notification using the given title and body templates.
// Automatically creates notification data, adding the given error
func (s *script) notify(titleTemplate string, bodyTemplate string, err error) error {
params := NotificationData{
Error: err,
Stats: s.stats,
Config: s.c,
}
titleBuf := &bytes.Buffer{}
if err := s.template.ExecuteTemplate(titleBuf, titleTemplate, params); err != nil {
return fmt.Errorf("notifyFailure: error executing %s template: %w", titleTemplate, err)
}
bodyBuf := &bytes.Buffer{}
if err := s.template.ExecuteTemplate(bodyBuf, bodyTemplate, params); err != nil {
return fmt.Errorf("notifyFailure: error executing %s template: %w", bodyTemplate, err)
}
if err := s.sendNotification(titleBuf.String(), bodyBuf.String()); err != nil {
return fmt.Errorf("notifyFailure: error notifying: %w", err)
}
return nil
}
// notifyFailure sends a notification about a failed backup run
func (s *script) notifyFailure(err error) error {
return s.notify("title_failure", "body_failure", err)
}
// notifyFailure sends a notification about a successful backup run
func (s *script) notifySuccess() error {
title := fmt.Sprintf("Success running docker-volume-backup at %s", s.start.Format(time.RFC3339))
body := fmt.Sprintf(
"Running docker-volume-backup succeeded.\n\nLog output was:\n\n%s\n", s.output.String(),
)
if err := s.sendNotification(title, body); err != nil {
return fmt.Errorf("notifySuccess: error notifying: %w", err)
}
return nil
return s.notify("title_success", "body_success", nil)
}
// sendNotification sends a notification to all configured third party services
@ -382,6 +475,12 @@ func (s *script) stopContainers() (func() error, error) {
)
}
s.stats.Containers = ContainersStats{
All: uint(len(allContainers)),
ToStop: uint(len(containersToStop)),
Stopped: uint(len(stoppedContainers)),
}
return func() error {
servicesRequiringUpdate := map[string]struct{}{}
@ -525,6 +624,17 @@ func (s *script) encryptBackup() error {
// as per the given configuration.
func (s *script) copyBackup() error {
_, name := path.Split(s.file)
if stat, err := os.Stat(s.file); err != nil {
return fmt.Errorf("copyBackup: unable to stat backup file: %w", err)
} else {
size := stat.Size()
s.stats.BackupFile = BackupFileStats{
Size: uint64(size),
Name: name,
FullPath: s.file,
}
}
if s.minioClient != nil {
if _, err := s.minioClient.FPutObject(context.Background(), s.c.AwsS3BucketName, filepath.Join(s.c.AwsS3Path, name), s.file, minio.PutObjectOptions{
ContentType: "application/tar+gzip",
@ -575,12 +685,7 @@ func (s *script) pruneOldBackups() error {
return nil
}
if s.c.BackupPruningLeeway != 0 {
s.logger.Infof("Sleeping for %s before pruning backups.", s.c.BackupPruningLeeway)
time.Sleep(s.c.BackupPruningLeeway)
}
deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays))
deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
// Prune minio/S3 backups
if s.minioClient != nil {
@ -604,6 +709,10 @@ func (s *script) pruneOldBackups() error {
}
}
s.stats.Storages.S3 = StorageStats{
Total: uint(lenCandidates),
Pruned: uint(len(matches)),
}
if len(matches) != 0 && len(matches) != lenCandidates {
objectsCh := make(chan minio.ObjectInfo)
go func() {
@ -619,6 +728,7 @@ func (s *script) pruneOldBackups() error {
removeErrors = append(removeErrors, result.Err)
}
}
s.stats.Storages.S3.PruneErrors = uint(len(removeErrors))
if len(removeErrors) != 0 {
return fmt.Errorf(
@ -627,10 +737,11 @@ func (s *script) pruneOldBackups() error {
join(removeErrors...),
)
}
s.logger.Infof(
"Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.",
len(matches),
lenCandidates,
s.stats.Storages.S3.Pruned,
s.stats.Storages.S3.Total,
s.c.BackupRetentionDays,
)
} else if len(matches) != 0 && len(matches) == lenCandidates {
@ -659,14 +770,33 @@ func (s *script) pruneOldBackups() error {
}
}
s.stats.Storages.WebDAV = StorageStats{
Total: uint(lenCandidates),
Pruned: uint(len(matches)),
}
if len(matches) != 0 && len(matches) != lenCandidates {
var removeErrors []error
for _, match := range matches {
if err := s.webdavClient.Remove(filepath.Join(s.c.WebdavPath, match.Name())); err != nil {
return fmt.Errorf("pruneOldBackups: error removing a file from remote storage: %w", err)
}
removeErrors = append(removeErrors, err)
} else {
s.logger.Infof("Pruned %s from WebDAV: %s", match.Name(), filepath.Join(s.c.WebdavUrl, s.c.WebdavPath))
}
s.logger.Infof("Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.", len(matches), lenCandidates, s.c.BackupRetentionDays)
}
s.stats.Storages.WebDAV.PruneErrors = uint(len(removeErrors))
if len(removeErrors) != 0 {
return fmt.Errorf(
"pruneOldBackups: %d error(s) removing files from remote storage: %w",
len(removeErrors),
join(removeErrors...),
)
}
s.logger.Infof(
"Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.",
s.stats.Storages.WebDAV.Pruned,
s.stats.Storages.WebDAV.Total,
s.c.BackupRetentionDays,
)
} else if len(matches) != 0 && len(matches) == lenCandidates {
s.logger.Warnf("The current configuration would delete all %d remote backup copies.", len(matches))
s.logger.Warn("Refusing to do so, please check your configuration.")
@ -721,6 +851,10 @@ func (s *script) pruneOldBackups() error {
}
}
s.stats.Storages.Local = StorageStats{
Total: uint(len(candidates)),
Pruned: uint(len(matches)),
}
if len(matches) != 0 && len(matches) != len(candidates) {
var removeErrors []error
for _, match := range matches {
@ -729,6 +863,7 @@ func (s *script) pruneOldBackups() error {
}
}
if len(removeErrors) != 0 {
s.stats.Storages.Local.PruneErrors = uint(len(removeErrors))
return fmt.Errorf(
"pruneOldBackups: %d error(s) deleting local files, starting with: %w",
len(removeErrors),
@ -737,8 +872,8 @@ func (s *script) pruneOldBackups() error {
}
s.logger.Infof(
"Pruned %d out of %d local backup(s) as their age exceeded the configured retention period of %d days.",
len(matches),
len(candidates),
s.stats.Storages.Local.Pruned,
s.stats.Storages.Local.Total,
s.c.BackupRetentionDays,
)
} else if len(matches) != 0 && len(matches) == len(candidates) {
@ -856,6 +991,26 @@ func join(errs ...error) error {
return errors.New("[" + strings.Join(msgs, ", ") + "]")
}
// formatBytes converts an amount of bytes in a human-readable representation
// the decimal parameter specifies if using powers of 1000 (decimal) or powers of 1024 (binary)
func formatBytes(b uint64, decimal bool) string {
unit := uint64(1024)
format := "%.1f %ciB"
if decimal {
unit = uint64(1000)
format = "%.1f %cB"
}
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := unit, 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf(format, float64(b)/float64(div), "kMGTPE"[exp])
}
// buffer takes an io.Writer and returns a wrapped version of the
// writer that writes to both the original target as well as the returned buffer
func buffer(w io.Writer) (io.Writer, *bytes.Buffer) {

26
cmd/backup/notifications.tmpl Normal file
View File

@ -0,0 +1,26 @@
{{ define "title_failure" -}}
Failure running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
{{- end }}
{{ define "body_failure" -}}
Running docker-volume-backup failed with error: {{ .Error }}
Log output of the failed run was:
{{ .Stats.LogOutput }}
{{- end }}
{{ define "title_success" -}}
Success running docker-volume-backup at {{ .Stats.StartTime | formatTime }}
{{- end }}
{{ define "body_success" -}}
Running docker-volume-backup succeeded.
Log output was:
{{ .Stats.LogOutput }}
{{- end }}

38
docs/NOTIFICATION-TEMPLATES.md Normal file
View File

@ -0,0 +1,38 @@
# Notification templates reference
In order to customize title and body of notifications you'll have to write a [go template](https://pkg.go.dev/text/template) and mount it inside the `/etc/dockervolumebackup/notifications.d/` directory.
Configuration, data about the backup run and helper functions will be passed to this template, this page documents them fully.
## Data
Here is a list of all data passed to the template:
* `Config`: this object holds the configuration that has been passed to the script. The field names are the name of the recognized environment variables converted in PascalCase. (e.g. `BACKUP_STOP_CONTAINER_LABEL` becomes `BackupStopContainerLabel`)
* `Error`: the error that made the backup fail. Only available in the `title_failure` and `body_failure` templates
* `Stats`: objects that holds stats regarding script execution. In case of an unsuccessful run, some information may not be available.
* `StartTime`: time when the script started execution
* `EndTime`: time when the backup has completed successfully (after pruning)
* `TookTime`: amount of time it took for the backup to run. (equal to `EndTime - StartTime`)
* `LogOutput`: full log of the application
* `Containers`: object containing stats about the docker containers
* `All`: total number of containers
* `ToStop`: number of containers matched by the stop rule
* `Stopped`: number of containers successfully stopped
* `StopErrors`: number of containers that were unable to be stopped (equal to `ToStop - Stopped`)
* `BackupFile`: object containing information about the backup file
* `Name`: name of the backup file (e.g. `backup-2022-02-11T01-00-00.tar.gz`)
* `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
* `Size`: size in bytes of the backup file
* `Storages`: object that holds stats about each storage
* `Local`, `S3` or `WebDAV`:
* `Total`: total number of backup files
* `Pruned`: number of backup files that were deleted due to pruning rule
* `PruneErrors`: number of backup files that were unable to be pruned
## Functions
Some formatting functions are also available:
* `formatTime`: formats a time object using [RFC3339](https://datatracker.ietf.org/doc/html/rfc3339) format (e.g. `2022-02-11T01:00:00Z`)
* `formatBytesBin`: formats an amount of bytes using powers of 1024 (e.g. `7055258` bytes will be `6.7 MiB`)
* `formatBytesDec`: formats an amount of bytes using powers of 1000 (e.g. `7055258` bytes will be `7.1 MB`)

1
test/swarm/run.sh
View File

@ -23,6 +23,7 @@ docker run --rm -it \
echo "[TEST:PASS] Found relevant files in untared backup."
sleep 5
if [ "$(docker ps -q | wc -l)" != "5" ]; then
echo "[TEST:FAIL] Expected all containers to be running post backup, instead seen:"
docker ps -a