mirror of
https://github.com/offen/docker-volume-backup.git
synced 2024-11-10 00:30:29 +01:00
Hoist control for exiting script a level up (#348)
* Hoist control for exiting script a level up * Do not accidentally nil out errors * Log when running schedule * Remove duplicate log line * Warn on cron schedule that will never run
This commit is contained in:
parent
69eceb3982
commit
65626dd3d4
@ -38,7 +38,7 @@ func loadConfig(lookup envProxy) (*Config, error) {
|
|||||||
|
|
||||||
var c = &Config{}
|
var c = &Config{}
|
||||||
if err := envconfig.Process("", c); err != nil {
|
if err := envconfig.Process("", c); err != nil {
|
||||||
return nil, fmt.Errorf("failed to process configuration values, error: %w", err)
|
return nil, fmt.Errorf("loadConfig: failed to process configuration values: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return c, nil
|
return c, nil
|
||||||
@ -48,33 +48,39 @@ func loadEnvVars() (*Config, error) {
|
|||||||
return loadConfig(os.LookupEnv)
|
return loadConfig(os.LookupEnv)
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadEnvFiles(directory string) ([]*Config, error) {
|
type configFile struct {
|
||||||
|
name string
|
||||||
|
config *Config
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadEnvFiles(directory string) ([]configFile, error) {
|
||||||
items, err := os.ReadDir(directory)
|
items, err := os.ReadDir(directory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return nil, fmt.Errorf("failed to read files from env directory, error: %w", err)
|
return nil, fmt.Errorf("loadEnvFiles: failed to read files from env directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var cs = make([]*Config, 0)
|
cs := []configFile{}
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
if !item.IsDir() {
|
if item.IsDir() {
|
||||||
p := filepath.Join(directory, item.Name())
|
continue
|
||||||
envFile, err := godotenv.Read(p)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error reading config file %s, error: %w", p, err)
|
|
||||||
}
|
|
||||||
lookup := func(key string) (string, bool) {
|
|
||||||
val, ok := envFile[key]
|
|
||||||
return val, ok
|
|
||||||
}
|
|
||||||
c, err := loadConfig(lookup)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error loading config from file %s, error: %w", p, err)
|
|
||||||
}
|
|
||||||
cs = append(cs, c)
|
|
||||||
}
|
}
|
||||||
|
p := filepath.Join(directory, item.Name())
|
||||||
|
envFile, err := godotenv.Read(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("loadEnvFiles: error reading config file %s: %w", p, err)
|
||||||
|
}
|
||||||
|
lookup := func(key string) (string, bool) {
|
||||||
|
val, ok := envFile[key]
|
||||||
|
return val, ok
|
||||||
|
}
|
||||||
|
c, err := loadConfig(lookup)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("loadEnvFiles: error loading config from file %s: %w", p, err)
|
||||||
|
}
|
||||||
|
cs = append(cs, configFile{config: c, name: item.Name()})
|
||||||
}
|
}
|
||||||
|
|
||||||
return cs, nil
|
return cs, nil
|
||||||
|
29
cmd/backup/cron.go
Normal file
29
cmd/backup/cron.go
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
// Copyright 2024 - Offen Authors <hioffen@posteo.de>
|
||||||
|
// SPDX-License-Identifier: MPL-2.0
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/robfig/cron/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
// checkCronSchedule detects whether the given cron expression will actually
|
||||||
|
// ever be executed or not.
|
||||||
|
func checkCronSchedule(expression string) (ok bool) {
|
||||||
|
defer func() {
|
||||||
|
if err := recover(); err != nil {
|
||||||
|
ok = false
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
sched, err := cron.ParseStandard(expression)
|
||||||
|
if err != nil {
|
||||||
|
ok = false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
sched.Next(now) // panics when the cron would never run
|
||||||
|
ok = true
|
||||||
|
return
|
||||||
|
}
|
@ -188,13 +188,18 @@ func (s *script) withLabeledCommands(step lifecyclePhase, cb func() error) func(
|
|||||||
if s.cli == nil {
|
if s.cli == nil {
|
||||||
return cb
|
return cb
|
||||||
}
|
}
|
||||||
return func() error {
|
return func() (err error) {
|
||||||
if err := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil {
|
if err = s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil {
|
||||||
return fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err)
|
err = fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
defer func() {
|
defer func() {
|
||||||
s.must(s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step)))
|
derr := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step))
|
||||||
|
if err == nil && derr != nil {
|
||||||
|
err = derr
|
||||||
|
}
|
||||||
}()
|
}()
|
||||||
return cb()
|
err = cb()
|
||||||
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -14,79 +14,115 @@ import (
|
|||||||
"github.com/robfig/cron/v3"
|
"github.com/robfig/cron/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
func runScript(c *Config) (ret error) {
|
type command struct {
|
||||||
s, err := newScript(c)
|
logger *slog.Logger
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
err = unlock()
|
|
||||||
if err != nil {
|
|
||||||
ret = err
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
if pArg := recover(); pArg != nil {
|
|
||||||
if err, ok := pArg.(error); ok {
|
|
||||||
s.logger.Error(
|
|
||||||
fmt.Sprintf("Executing the script encountered a panic: %v", err),
|
|
||||||
)
|
|
||||||
if hookErr := s.runHooks(err); hookErr != nil {
|
|
||||||
s.logger.Error(
|
|
||||||
fmt.Sprintf("An error occurred calling the registered hooks: %s", hookErr),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
ret = err
|
|
||||||
} else {
|
|
||||||
s.logger.Error(
|
|
||||||
fmt.Sprintf("Executing the script encountered an unrecoverable panic: %v", err),
|
|
||||||
)
|
|
||||||
|
|
||||||
panic(pArg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.runHooks(nil); err != nil {
|
|
||||||
s.logger.Error(
|
|
||||||
fmt.Sprintf(
|
|
||||||
"Backup procedure ran successfully, but an error ocurred calling the registered hooks: %v",
|
|
||||||
err,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
ret = err
|
|
||||||
}
|
|
||||||
s.logger.Info("Finished running backup tasks.")
|
|
||||||
}()
|
|
||||||
|
|
||||||
s.must(s.withLabeledCommands(lifecyclePhaseArchive, func() error {
|
|
||||||
restartContainersAndServices, err := s.stopContainersAndServices()
|
|
||||||
// The mechanism for restarting containers is not using hooks as it
|
|
||||||
// should happen as soon as possible (i.e. before uploading backups or
|
|
||||||
// similar).
|
|
||||||
defer func() {
|
|
||||||
s.must(restartContainersAndServices())
|
|
||||||
}()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return s.createArchive()
|
|
||||||
})())
|
|
||||||
|
|
||||||
s.must(s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)())
|
|
||||||
s.must(s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)())
|
|
||||||
s.must(s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)())
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func runInForeground() error {
|
func newCommand() *command {
|
||||||
|
return &command{
|
||||||
|
logger: slog.New(slog.NewTextHandler(os.Stdout, nil)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *command) must(err error) {
|
||||||
|
if err != nil {
|
||||||
|
c.logger.Error(
|
||||||
|
fmt.Sprintf("Fatal error running command: %v", err),
|
||||||
|
"error",
|
||||||
|
err,
|
||||||
|
)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runScript(c *Config) (err error) {
|
||||||
|
defer func() {
|
||||||
|
if derr := recover(); derr != nil {
|
||||||
|
err = fmt.Errorf("runScript: unexpected panic running script: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
s, err := newScript(c)
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("runScript: error instantiating script: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
runErr := func() (err error) {
|
||||||
|
unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("runScript: error acquiring file lock: %w", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
derr := unlock()
|
||||||
|
if err == nil && derr != nil {
|
||||||
|
err = fmt.Errorf("runScript: error releasing file lock: %w", derr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
scriptErr := func() error {
|
||||||
|
if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) {
|
||||||
|
restartContainersAndServices, err := s.stopContainersAndServices()
|
||||||
|
// The mechanism for restarting containers is not using hooks as it
|
||||||
|
// should happen as soon as possible (i.e. before uploading backups or
|
||||||
|
// similar).
|
||||||
|
defer func() {
|
||||||
|
derr := restartContainersAndServices()
|
||||||
|
if err == nil {
|
||||||
|
err = derr
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = s.createArchive()
|
||||||
|
return
|
||||||
|
})(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}()
|
||||||
|
|
||||||
|
if hookErr := s.runHooks(scriptErr); hookErr != nil {
|
||||||
|
if scriptErr != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"runScript: error %w executing the script followed by %w calling the registered hooks",
|
||||||
|
scriptErr,
|
||||||
|
hookErr,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return fmt.Errorf(
|
||||||
|
"runScript: the script ran successfully, but an error occurred calling the registered hooks: %w",
|
||||||
|
hookErr,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if scriptErr != nil {
|
||||||
|
return fmt.Errorf("runScript: error running script: %w", scriptErr)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}()
|
||||||
|
|
||||||
|
if runErr != nil {
|
||||||
|
s.logger.Error(
|
||||||
|
fmt.Sprintf("Script run failed: %v", runErr), "error", runErr,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return runErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *command) runInForeground() error {
|
||||||
cr := cron.New(
|
cr := cron.New(
|
||||||
cron.WithParser(
|
cron.WithParser(
|
||||||
cron.NewParser(
|
cron.NewParser(
|
||||||
@ -95,36 +131,60 @@ func runInForeground() error {
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
addJob := func(c *Config) error {
|
addJob := func(config *Config, name string) error {
|
||||||
_, err := cr.AddFunc(c.BackupCronExpression, func() {
|
if _, err := cr.AddFunc(config.BackupCronExpression, func() {
|
||||||
err := runScript(c)
|
c.logger.Info(
|
||||||
if err != nil {
|
fmt.Sprintf(
|
||||||
slog.Error("unexpected error during backup", "error", err)
|
"Now running script on schedule %s",
|
||||||
|
config.BackupCronExpression,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if err := runScript(config); err != nil {
|
||||||
|
c.logger.Error(
|
||||||
|
fmt.Sprintf(
|
||||||
|
"Unexpected error running schedule %s: %v",
|
||||||
|
config.BackupCronExpression,
|
||||||
|
err,
|
||||||
|
),
|
||||||
|
"error",
|
||||||
|
err,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
})
|
}); err != nil {
|
||||||
return err
|
return fmt.Errorf("addJob: error adding schedule %s: %w", config.BackupCronExpression, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.logger.Info(fmt.Sprintf("Successfully scheduled backup %s with expression %s", name, config.BackupCronExpression))
|
||||||
|
if ok := checkCronSchedule(config.BackupCronExpression); !ok {
|
||||||
|
c.logger.Warn(
|
||||||
|
fmt.Sprintf("Scheduled cron expression %s will never run, is this intentional?", config.BackupCronExpression),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
cs, err := loadEnvFiles("/etc/dockervolumebackup/conf.d")
|
cs, err := loadEnvFiles("/etc/dockervolumebackup/conf.d")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !os.IsNotExist(err) {
|
if !os.IsNotExist(err) {
|
||||||
return fmt.Errorf("could not load config from environment files, error: %w", err)
|
return fmt.Errorf("runInForeground: could not load config from environment files: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
c, err := loadEnvVars()
|
c, err := loadEnvVars()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not load config from environment variables")
|
return fmt.Errorf("runInForeground: could not load config from environment variables: %w", err)
|
||||||
} else {
|
} else {
|
||||||
err = addJob(c)
|
err = addJob(c, "from environment")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not add cron job, error: %w", err)
|
return fmt.Errorf("runInForeground: error adding job from env: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for _, c := range cs {
|
c.logger.Info("/etc/dockervolumebackup/conf.d was found, using configuration files from this directory.")
|
||||||
err = addJob(c)
|
for _, config := range cs {
|
||||||
|
err = addJob(config.config, config.name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not add cron job, error: %w", err)
|
return fmt.Errorf("runInForeground: error adding jobs from conf files: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -139,33 +199,27 @@ func runInForeground() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runAsCommand() error {
|
func (c *command) runAsCommand() error {
|
||||||
c, err := loadEnvVars()
|
config, err := loadEnvVars()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not load config from environment variables, error: %w", err)
|
return fmt.Errorf("runAsCommand: error loading env vars: %w", err)
|
||||||
}
|
}
|
||||||
|
err = runScript(config)
|
||||||
err = runScript(c)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unexpected error during backup, error: %w", err)
|
return fmt.Errorf("runAsCommand: error running script: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
serve := flag.Bool("foreground", false, "run the tool in the foreground")
|
foreground := flag.Bool("foreground", false, "run the tool in the foreground")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
var err error
|
c := newCommand()
|
||||||
if *serve {
|
if *foreground {
|
||||||
err = runInForeground()
|
c.must(c.runInForeground())
|
||||||
} else {
|
} else {
|
||||||
err = runAsCommand()
|
c.must(c.runAsCommand())
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
slog.Error("ran into an issue during execution", "error", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -480,17 +480,6 @@ func (s *script) pruneBackups() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// must exits the script run prematurely in case the given error
|
|
||||||
// is non-nil.
|
|
||||||
func (s *script) must(err error) {
|
|
||||||
if err != nil {
|
|
||||||
s.logger.Error(
|
|
||||||
fmt.Sprintf("Fatal error running backup: %s", err),
|
|
||||||
)
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// skipPrune returns true if the given backend name is contained in the
|
// skipPrune returns true if the given backend name is contained in the
|
||||||
// list of skipped backends.
|
// list of skipped backends.
|
||||||
func skipPrune(name string, skippedBackends []string) bool {
|
func skipPrune(name string, skippedBackends []string) bool {
|
||||||
|
@ -13,6 +13,8 @@ docker compose up -d --quiet-pull
|
|||||||
# sleep until a backup is guaranteed to have happened on the 1 minute schedule
|
# sleep until a backup is guaranteed to have happened on the 1 minute schedule
|
||||||
sleep 100
|
sleep 100
|
||||||
|
|
||||||
|
docker compose logs backup
|
||||||
|
|
||||||
if [ ! -f "$LOCAL_DIR/conf.tar.gz" ]; then
|
if [ ! -f "$LOCAL_DIR/conf.tar.gz" ]; then
|
||||||
fail "Config from file was not used."
|
fail "Config from file was not used."
|
||||||
fi
|
fi
|
||||||
|
Loading…
Reference in New Issue
Block a user