offen/docker-volume-backup
2
0
Fork 0
mirror of https://github.com/offen/docker-volume-backup.git synced 2024-11-21 21:10:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Move cron scheduling inside application (#338)

Browse Source 
* Move cron scheduling inside application

* Make envvar a fallback and check for errors

* Panic significantly less

* propagate error out of runBackup

* Add structured logging

* FIx error propagation to exit

* Enable the new scheduler by default

* Review fixes

* Added docs and better error propagation
This commit is contained in:
pixxon 2024-02-06 21:05:38 +01:00 committed by Frederik Ring
parent 64d934102d
commit 1d45062100
8 changed files with 218 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -18,4 +18,4 @@ RUN apk add --no-cache ca-certificates
COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
COPY --chmod=755 ./entrypoint.sh /root/
ENTRYPOINT ["/root/entrypoint.sh"]
ENTRYPOINT ["/usr/bin/backup", "-foreground"]

1
cmd/backup/config.go
View File

@ -34,6 +34,7 @@ type Config struct {
BackupFilenameExpand bool `split_words:"true"`
BackupLatestSymlink string `split_words:"true"`
BackupArchive string `split_words:"true" default:"/archive"`
BackupCronExpression string `split_words:"true" default:"@daily"`
BackupRetentionDays int32 `split_words:"true" default:"-1"`
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
BackupPruningPrefix string `split_words:"true"`

81
cmd/backup/config_provider.go Normal file
View File

@ -0,0 +1,81 @@
// Copyright 2021-2022 - Offen Authors <hioffen@posteo.de>
// SPDX-License-Identifier: MPL-2.0
package main
import (
"fmt"
"os"
"path/filepath"
"github.com/joho/godotenv"
"github.com/offen/envconfig"
)
// envProxy is a function that mimics os.LookupEnv but can read values from any other source
type envProxy func(string) (string, bool)
func loadConfig(lookup envProxy) (*Config, error) {
envconfig.Lookup = func(key string) (string, bool) {
value, okValue := lookup(key)
location, okFile := lookup(key + "_FILE")
switch {
case okValue && !okFile: // only value
return value, true
case !okValue && okFile: // only file
contents, err := os.ReadFile(location)
if err != nil {
return "", false
}
return string(contents), true
case okValue && okFile: // both
return "", false
default: // neither, ignore
return "", false
}
}
var c = &Config{}
if err := envconfig.Process("", c); err != nil {
return nil, fmt.Errorf("failed to process configuration values, error: %w", err)
}
return c, nil
}
func loadEnvVars() (*Config, error) {
return loadConfig(os.LookupEnv)
}
func loadEnvFiles(directory string) ([]*Config, error) {
items, err := os.ReadDir(directory)
if err != nil {
if os.IsNotExist(err) {
return nil, err
}
return nil, fmt.Errorf("failed to read files from env directory, error: %w", err)
}
var cs = make([]*Config, 0)
for _, item := range items {
if !item.IsDir() {
p := filepath.Join(directory, item.Name())
envFile, err := godotenv.Read(p)
if err != nil {
return nil, fmt.Errorf("error reading config file %s, error: %w", p, err)
}
lookup := func(key string) (string, bool) {
val, ok := envFile[key]
return val, ok
}
c, err := loadConfig(lookup)
if err != nil {
return nil, fmt.Errorf("error loading config from file %s, error: %w", p, err)
}
cs = append(cs, c)
}
}
return cs, nil
}

119
cmd/backup/main.go
View File

@ -4,21 +4,33 @@
package main
import (
"flag"
"fmt"
"log/slog"
"os"
"os/signal"
"syscall"
"github.com/robfig/cron/v3"
)
func main() {
s, err := newScript()
func runScript(c *Config) (ret error) {
s, err := newScript(c)
if err != nil {
panic(err)
return err
}
unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
if err != nil {
return err
}
defer func() {
s.must(unlock())
err = unlock()
if err != nil {
ret = err
}
}()
s.must(err)
defer func() {
if pArg := recover(); pArg != nil {
@ -31,9 +43,14 @@ func main() {
fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr),
)
}
os.Exit(1)
ret = err
} else {
s.logger.Error(
fmt.Sprintf("Executing the script encountered an unrecoverable panic: %v", err),
)
panic(pArg)
}
panic(pArg)
}
if err := s.runHooks(nil); err != nil {
@ -43,7 +60,7 @@ func main() {
err,
),
)
os.Exit(1)
ret = err
}
s.logger.Info("Finished running backup tasks.")
}()
@ -65,4 +82,90 @@ func main() {
s.must(s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)())
s.must(s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)())
s.must(s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)())
return nil
}
func runInForeground() error {
cr := cron.New(
cron.WithParser(
cron.NewParser(
cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
),
),
)
addJob := func(c *Config) error {
_, err := cr.AddFunc(c.BackupCronExpression, func() {
err := runScript(c)
if err != nil {
slog.Error("unexpected error during backup", "error", err)
}
})
return err
}
cs, err := loadEnvFiles("/etc/dockervolumebackup/conf.d")
if err != nil {
if !os.IsNotExist(err) {
return fmt.Errorf("could not load config from environment files, error: %w", err)
}
c, err := loadEnvVars()
if err != nil {
return fmt.Errorf("could not load config from environment variables")
} else {
err = addJob(c)
if err != nil {
return fmt.Errorf("could not add cron job, error: %w", err)
}
}
} else {
for _, c := range cs {
err = addJob(c)
if err != nil {
return fmt.Errorf("could not add cron job, error: %w", err)
}
}
}
var quit = make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT)
cr.Start()
<-quit
ctx := cr.Stop()
<-ctx.Done()
return nil
}
func runAsCommand() error {
c, err := loadEnvVars()
if err != nil {
return fmt.Errorf("could not load config from environment variables, error: %w", err)
}
err = runScript(c)
if err != nil {
return fmt.Errorf("unexpected error during backup, error: %w", err)
}
return nil
}
func main() {
serve := flag.Bool("foreground", false, "run the tool in the foreground")
flag.Parse()
var err error
if *serve {
err = runInForeground()
} else {
err = runAsCommand()
}
if err != nil {
slog.Error("ran into an issue during execution", "error", err)
os.Exit(1)
}
}

31
cmd/backup/script.go
View File

@ -30,7 +30,6 @@ import (
"github.com/containrrr/shoutrrr/pkg/router"
"github.com/docker/docker/client"
"github.com/leekchan/timeutil"
"github.com/offen/envconfig"
"github.com/otiai10/copy"
"golang.org/x/sync/errgroup"
)
@ -58,10 +57,10 @@ type script struct {
// remote resources like the Docker engine or remote storage locations. All
// reading from env vars or other configuration sources is expected to happen
// in this method.
func newScript() (*script, error) {
func newScript(c *Config) (*script, error) {
stdOut, logBuffer := buffer(os.Stdout)
s := &script{
c: &Config{},
c: c,
logger: slog.New(slog.NewTextHandler(stdOut, nil)),
stats: &Stats{
StartTime: time.Now(),
@ -83,32 +82,6 @@ func newScript() (*script, error) {
return nil
})
envconfig.Lookup = func(key string) (string, bool) {
value, okValue := os.LookupEnv(key)
location, okFile := os.LookupEnv(key + "_FILE")
switch {
case okValue && !okFile: // only value
return value, true
case !okValue && okFile: // only file
contents, err := os.ReadFile(location)
if err != nil {
s.must(fmt.Errorf("newScript: failed to read %s! Error: %s", location, err))
return "", false
}
return string(contents), true
case okValue && okFile: // both
s.must(fmt.Errorf("newScript: both %s and %s are set!", key, key+"_FILE"))
return "", false
default: // neither, ignore
return "", false
}
}
if err := envconfig.Process("", s.c); err != nil {
return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
}
s.file = path.Join("/tmp", s.c.BackupFilename)
tmplFileName, tErr := template.New("extension").Parse(s.file)

19
docs/reference/index.md
View File

@ -23,9 +23,22 @@ You can populate below template according to your requirements and use it as you
```
########### BACKUP SCHEDULE
# Backups run on the given cron schedule in `busybox` flavor. If no
# value is set, `@daily` will be used. If you do not want the cron
# to ever run, use `0 0 5 31 2 ?`.
# A cron expression represents a set of times, using 5 or 6 space-separated fields.
#
# Field name | Mandatory? | Allowed values | Allowed special characters
# ---------- | ---------- | -------------- | --------------------------
# Seconds | No | 0-59 | * / , -
# Minutes | Yes | 0-59 | * / , -
# Hours | Yes | 0-23 | * / , -
# Day of month | Yes | 1-31 | * / , - ?
# Month | Yes | 1-12 or JAN-DEC | * / , -
# Day of week | Yes | 0-6 or SUN-SAT | * / , - ?
#
# Month and Day-of-week field values are case insensitive.
# "SUN", "Sun", and "sun" are equally accepted.
# If no value is set, `@daily` will be used.
# If you do not want the cron to ever run, use `0 0 5 31 2 ?`.
# BACKUP_CRON_EXPRESSION="0 2 * * *"

2
go.mod
View File

@ -28,6 +28,8 @@ require (
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
github.com/golang/protobuf v1.5.3 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/robfig/cron/v3 v3.0.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.31.0 // indirect
)

4
go.sum
View File

@ -443,6 +443,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc=
github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -593,6 +595,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=