mirror of
https://github.com/offen/docker-volume-backup.git
synced 2024-11-10 00:30:29 +01:00
Move cron scheduling inside application (#338)
* Move cron scheduling inside application * Make envvar a fallback and check for errors * Panic significantly less * propagate error out of runBackup * Add structured logging * FIx error propagation to exit * Enable the new scheduler by default * Review fixes * Added docs and better error propagation
This commit is contained in:
parent
64d934102d
commit
1d45062100
@ -18,4 +18,4 @@ RUN apk add --no-cache ca-certificates
|
|||||||
COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
|
COPY --from=builder /app/cmd/backup/backup /usr/bin/backup
|
||||||
COPY --chmod=755 ./entrypoint.sh /root/
|
COPY --chmod=755 ./entrypoint.sh /root/
|
||||||
|
|
||||||
ENTRYPOINT ["/root/entrypoint.sh"]
|
ENTRYPOINT ["/usr/bin/backup", "-foreground"]
|
||||||
|
@ -34,6 +34,7 @@ type Config struct {
|
|||||||
BackupFilenameExpand bool `split_words:"true"`
|
BackupFilenameExpand bool `split_words:"true"`
|
||||||
BackupLatestSymlink string `split_words:"true"`
|
BackupLatestSymlink string `split_words:"true"`
|
||||||
BackupArchive string `split_words:"true" default:"/archive"`
|
BackupArchive string `split_words:"true" default:"/archive"`
|
||||||
|
BackupCronExpression string `split_words:"true" default:"@daily"`
|
||||||
BackupRetentionDays int32 `split_words:"true" default:"-1"`
|
BackupRetentionDays int32 `split_words:"true" default:"-1"`
|
||||||
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
|
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
|
||||||
BackupPruningPrefix string `split_words:"true"`
|
BackupPruningPrefix string `split_words:"true"`
|
||||||
|
81
cmd/backup/config_provider.go
Normal file
81
cmd/backup/config_provider.go
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
// Copyright 2021-2022 - Offen Authors <hioffen@posteo.de>
|
||||||
|
// SPDX-License-Identifier: MPL-2.0
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/joho/godotenv"
|
||||||
|
"github.com/offen/envconfig"
|
||||||
|
)
|
||||||
|
|
||||||
|
// envProxy is a function that mimics os.LookupEnv but can read values from any other source
|
||||||
|
type envProxy func(string) (string, bool)
|
||||||
|
|
||||||
|
func loadConfig(lookup envProxy) (*Config, error) {
|
||||||
|
envconfig.Lookup = func(key string) (string, bool) {
|
||||||
|
value, okValue := lookup(key)
|
||||||
|
location, okFile := lookup(key + "_FILE")
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case okValue && !okFile: // only value
|
||||||
|
return value, true
|
||||||
|
case !okValue && okFile: // only file
|
||||||
|
contents, err := os.ReadFile(location)
|
||||||
|
if err != nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return string(contents), true
|
||||||
|
case okValue && okFile: // both
|
||||||
|
return "", false
|
||||||
|
default: // neither, ignore
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var c = &Config{}
|
||||||
|
if err := envconfig.Process("", c); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to process configuration values, error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadEnvVars() (*Config, error) {
|
||||||
|
return loadConfig(os.LookupEnv)
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadEnvFiles(directory string) ([]*Config, error) {
|
||||||
|
items, err := os.ReadDir(directory)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("failed to read files from env directory, error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var cs = make([]*Config, 0)
|
||||||
|
for _, item := range items {
|
||||||
|
if !item.IsDir() {
|
||||||
|
p := filepath.Join(directory, item.Name())
|
||||||
|
envFile, err := godotenv.Read(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error reading config file %s, error: %w", p, err)
|
||||||
|
}
|
||||||
|
lookup := func(key string) (string, bool) {
|
||||||
|
val, ok := envFile[key]
|
||||||
|
return val, ok
|
||||||
|
}
|
||||||
|
c, err := loadConfig(lookup)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error loading config from file %s, error: %w", p, err)
|
||||||
|
}
|
||||||
|
cs = append(cs, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return cs, nil
|
||||||
|
}
|
@ -4,21 +4,33 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/robfig/cron/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func runScript(c *Config) (ret error) {
|
||||||
s, err := newScript()
|
s, err := newScript(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
|
unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
s.must(unlock())
|
err = unlock()
|
||||||
|
if err != nil {
|
||||||
|
ret = err
|
||||||
|
}
|
||||||
}()
|
}()
|
||||||
s.must(err)
|
|
||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if pArg := recover(); pArg != nil {
|
if pArg := recover(); pArg != nil {
|
||||||
@ -31,10 +43,15 @@ func main() {
|
|||||||
fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr),
|
fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
os.Exit(1)
|
ret = err
|
||||||
}
|
} else {
|
||||||
|
s.logger.Error(
|
||||||
|
fmt.Sprintf("Executing the script encountered an unrecoverable panic: %v", err),
|
||||||
|
)
|
||||||
|
|
||||||
panic(pArg)
|
panic(pArg)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := s.runHooks(nil); err != nil {
|
if err := s.runHooks(nil); err != nil {
|
||||||
s.logger.Error(
|
s.logger.Error(
|
||||||
@ -43,7 +60,7 @@ func main() {
|
|||||||
err,
|
err,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
os.Exit(1)
|
ret = err
|
||||||
}
|
}
|
||||||
s.logger.Info("Finished running backup tasks.")
|
s.logger.Info("Finished running backup tasks.")
|
||||||
}()
|
}()
|
||||||
@ -65,4 +82,90 @@ func main() {
|
|||||||
s.must(s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)())
|
s.must(s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)())
|
||||||
s.must(s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)())
|
s.must(s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)())
|
||||||
s.must(s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)())
|
s.must(s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)())
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runInForeground() error {
|
||||||
|
cr := cron.New(
|
||||||
|
cron.WithParser(
|
||||||
|
cron.NewParser(
|
||||||
|
cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
addJob := func(c *Config) error {
|
||||||
|
_, err := cr.AddFunc(c.BackupCronExpression, func() {
|
||||||
|
err := runScript(c)
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("unexpected error during backup", "error", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
cs, err := loadEnvFiles("/etc/dockervolumebackup/conf.d")
|
||||||
|
if err != nil {
|
||||||
|
if !os.IsNotExist(err) {
|
||||||
|
return fmt.Errorf("could not load config from environment files, error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c, err := loadEnvVars()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not load config from environment variables")
|
||||||
|
} else {
|
||||||
|
err = addJob(c)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not add cron job, error: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for _, c := range cs {
|
||||||
|
err = addJob(c)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not add cron job, error: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var quit = make(chan os.Signal, 1)
|
||||||
|
signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT)
|
||||||
|
cr.Start()
|
||||||
|
<-quit
|
||||||
|
ctx := cr.Stop()
|
||||||
|
<-ctx.Done()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runAsCommand() error {
|
||||||
|
c, err := loadEnvVars()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("could not load config from environment variables, error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = runScript(c)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("unexpected error during backup, error: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
serve := flag.Bool("foreground", false, "run the tool in the foreground")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
var err error
|
||||||
|
if *serve {
|
||||||
|
err = runInForeground()
|
||||||
|
} else {
|
||||||
|
err = runAsCommand()
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("ran into an issue during execution", "error", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -30,7 +30,6 @@ import (
|
|||||||
"github.com/containrrr/shoutrrr/pkg/router"
|
"github.com/containrrr/shoutrrr/pkg/router"
|
||||||
"github.com/docker/docker/client"
|
"github.com/docker/docker/client"
|
||||||
"github.com/leekchan/timeutil"
|
"github.com/leekchan/timeutil"
|
||||||
"github.com/offen/envconfig"
|
|
||||||
"github.com/otiai10/copy"
|
"github.com/otiai10/copy"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
@ -58,10 +57,10 @@ type script struct {
|
|||||||
// remote resources like the Docker engine or remote storage locations. All
|
// remote resources like the Docker engine or remote storage locations. All
|
||||||
// reading from env vars or other configuration sources is expected to happen
|
// reading from env vars or other configuration sources is expected to happen
|
||||||
// in this method.
|
// in this method.
|
||||||
func newScript() (*script, error) {
|
func newScript(c *Config) (*script, error) {
|
||||||
stdOut, logBuffer := buffer(os.Stdout)
|
stdOut, logBuffer := buffer(os.Stdout)
|
||||||
s := &script{
|
s := &script{
|
||||||
c: &Config{},
|
c: c,
|
||||||
logger: slog.New(slog.NewTextHandler(stdOut, nil)),
|
logger: slog.New(slog.NewTextHandler(stdOut, nil)),
|
||||||
stats: &Stats{
|
stats: &Stats{
|
||||||
StartTime: time.Now(),
|
StartTime: time.Now(),
|
||||||
@ -83,32 +82,6 @@ func newScript() (*script, error) {
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
envconfig.Lookup = func(key string) (string, bool) {
|
|
||||||
value, okValue := os.LookupEnv(key)
|
|
||||||
location, okFile := os.LookupEnv(key + "_FILE")
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case okValue && !okFile: // only value
|
|
||||||
return value, true
|
|
||||||
case !okValue && okFile: // only file
|
|
||||||
contents, err := os.ReadFile(location)
|
|
||||||
if err != nil {
|
|
||||||
s.must(fmt.Errorf("newScript: failed to read %s! Error: %s", location, err))
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
return string(contents), true
|
|
||||||
case okValue && okFile: // both
|
|
||||||
s.must(fmt.Errorf("newScript: both %s and %s are set!", key, key+"_FILE"))
|
|
||||||
return "", false
|
|
||||||
default: // neither, ignore
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := envconfig.Process("", s.c); err != nil {
|
|
||||||
return nil, fmt.Errorf("newScript: failed to process configuration values: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.file = path.Join("/tmp", s.c.BackupFilename)
|
s.file = path.Join("/tmp", s.c.BackupFilename)
|
||||||
|
|
||||||
tmplFileName, tErr := template.New("extension").Parse(s.file)
|
tmplFileName, tErr := template.New("extension").Parse(s.file)
|
||||||
|
@ -23,9 +23,22 @@ You can populate below template according to your requirements and use it as you
|
|||||||
```
|
```
|
||||||
########### BACKUP SCHEDULE
|
########### BACKUP SCHEDULE
|
||||||
|
|
||||||
# Backups run on the given cron schedule in `busybox` flavor. If no
|
|
||||||
# value is set, `@daily` will be used. If you do not want the cron
|
# A cron expression represents a set of times, using 5 or 6 space-separated fields.
|
||||||
# to ever run, use `0 0 5 31 2 ?`.
|
#
|
||||||
|
# Field name | Mandatory? | Allowed values | Allowed special characters
|
||||||
|
# ---------- | ---------- | -------------- | --------------------------
|
||||||
|
# Seconds | No | 0-59 | * / , -
|
||||||
|
# Minutes | Yes | 0-59 | * / , -
|
||||||
|
# Hours | Yes | 0-23 | * / , -
|
||||||
|
# Day of month | Yes | 1-31 | * / , - ?
|
||||||
|
# Month | Yes | 1-12 or JAN-DEC | * / , -
|
||||||
|
# Day of week | Yes | 0-6 or SUN-SAT | * / , - ?
|
||||||
|
#
|
||||||
|
# Month and Day-of-week field values are case insensitive.
|
||||||
|
# "SUN", "Sun", and "sun" are equally accepted.
|
||||||
|
# If no value is set, `@daily` will be used.
|
||||||
|
# If you do not want the cron to ever run, use `0 0 5 31 2 ?`.
|
||||||
|
|
||||||
# BACKUP_CRON_EXPRESSION="0 2 * * *"
|
# BACKUP_CRON_EXPRESSION="0 2 * * *"
|
||||||
|
|
||||||
|
2
go.mod
2
go.mod
@ -28,6 +28,8 @@ require (
|
|||||||
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
|
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
|
||||||
github.com/golang/protobuf v1.5.3 // indirect
|
github.com/golang/protobuf v1.5.3 // indirect
|
||||||
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
|
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
|
||||||
|
github.com/joho/godotenv v1.5.1 // indirect
|
||||||
|
github.com/robfig/cron/v3 v3.0.0 // indirect
|
||||||
google.golang.org/appengine v1.6.7 // indirect
|
google.golang.org/appengine v1.6.7 // indirect
|
||||||
google.golang.org/protobuf v1.31.0 // indirect
|
google.golang.org/protobuf v1.31.0 // indirect
|
||||||
)
|
)
|
||||||
|
4
go.sum
4
go.sum
@ -443,6 +443,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
|
|||||||
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc=
|
github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc=
|
||||||
github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk=
|
github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk=
|
||||||
|
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||||
|
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||||
@ -593,6 +595,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
|
|||||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||||
|
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
|
||||||
|
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||||
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||||
|
Loading…
Reference in New Issue
Block a user