From 83fa0aae48dd1728bb3d824e2a0156fa3f5bfc92 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Tue, 13 Feb 2024 19:21:57 +0100 Subject: [PATCH] Refactor handling of runtime configuration to prepare for reloading --- cmd/backup/command.go | 155 ++++++++++++++++++++ cmd/backup/config.go | 39 +++++ cmd/backup/config_provider.go | 55 +++++-- cmd/backup/copy_archive.go | 41 ++++++ cmd/backup/create_archive.go | 87 +++++++++++ cmd/backup/cron.go | 29 ---- cmd/backup/encrypt_archive.go | 63 ++++++++ cmd/backup/exec.go | 8 +- cmd/backup/main.go | 227 +--------------------------- cmd/backup/profile.go | 24 +++ cmd/backup/prune_backups.go | 65 ++++++++ cmd/backup/run_script.go | 101 +++++++++++++ cmd/backup/script.go | 269 +++------------------------------- cmd/backup/util.go | 22 +++ 14 files changed, 665 insertions(+), 520 deletions(-) create mode 100644 cmd/backup/command.go create mode 100644 cmd/backup/copy_archive.go create mode 100644 cmd/backup/create_archive.go delete mode 100644 cmd/backup/cron.go create mode 100644 cmd/backup/encrypt_archive.go create mode 100644 cmd/backup/profile.go create mode 100644 cmd/backup/prune_backups.go create mode 100644 cmd/backup/run_script.go diff --git a/cmd/backup/command.go b/cmd/backup/command.go new file mode 100644 index 0000000..cdaac0f --- /dev/null +++ b/cmd/backup/command.go @@ -0,0 +1,155 @@ +// Copyright 2024 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package main + +import ( + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/robfig/cron/v3" +) + +type command struct { + logger *slog.Logger + schedules []cron.EntryID + cr *cron.Cron + reload chan struct{} +} + +func newCommand() *command { + return &command{ + logger: slog.New(slog.NewTextHandler(os.Stdout, nil)), + } +} + +// runAsCommand executes a backup run for each configuration that is available +// and then returns +func (c *command) runAsCommand() error { + configurations, err := sourceConfiguration(configStrategyEnv) + if err != nil { + return fmt.Errorf("runAsCommand: error loading env vars: %w", err) + } + + for _, config := range configurations { + if err := runScript(config); err != nil { + return fmt.Errorf("runAsCommand: error running script: %w", err) + } + } + + return nil +} + +type foregroundOpts struct { + profileCronExpression string +} + +// runInForeground starts the program as a long running process, scheduling +// a job for each configuration that is available. +func (c *command) runInForeground(opts foregroundOpts) error { + c.cr = cron.New( + cron.WithParser( + cron.NewParser( + cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor, + ), + ), + ) + + if err := c.schedule(configStrategyConfd); err != nil { + return fmt.Errorf("runInForeground: error scheduling: %w", err) + } + + if opts.profileCronExpression != "" { + if _, err := c.cr.AddFunc(opts.profileCronExpression, c.profile); err != nil { + return fmt.Errorf("runInForeground: error adding profiling job: %w", err) + } + } + + var quit = make(chan os.Signal, 1) + c.reload = make(chan struct{}, 1) + signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT) + c.cr.Start() + + for { + select { + case <-quit: + ctx := c.cr.Stop() + <-ctx.Done() + return nil + case <-c.reload: + if err := c.schedule(configStrategyConfd); err != nil { + return fmt.Errorf("runInForeground: error reloading configuration: %w", err) + } + } + } +} + +// schedule wipes all existing schedules and enqueues all schedules available +// using the given configuration strategy +func (c *command) schedule(strategy configStrategy) error { + for _, id := range c.schedules { + c.cr.Remove(id) + } + + configurations, err := sourceConfiguration(strategy) + if err != nil { + return fmt.Errorf("schedule: error sourcing configuration: %w", err) + } + + for _, cfg := range configurations { + config := cfg + id, err := c.cr.AddFunc(config.BackupCronExpression, func() { + c.logger.Info( + fmt.Sprintf( + "Now running script on schedule %s", + config.BackupCronExpression, + ), + ) + + if err := runScript(config); err != nil { + c.logger.Error( + fmt.Sprintf( + "Unexpected error running schedule %s: %v", + config.BackupCronExpression, + err, + ), + "error", + err, + ) + } + }) + + if err != nil { + return fmt.Errorf("addJob: error adding schedule %s: %w", config.BackupCronExpression, err) + } + c.logger.Info(fmt.Sprintf("Successfully scheduled backup %s with expression %s", config.source, config.BackupCronExpression)) + if ok := checkCronSchedule(config.BackupCronExpression); !ok { + c.logger.Warn( + fmt.Sprintf("Scheduled cron expression %s will never run, is this intentional?", config.BackupCronExpression), + ) + + if err != nil { + return fmt.Errorf("schedule: error scheduling: %w", err) + } + c.schedules = append(c.schedules, id) + } + } + + return nil +} + +// must exits the program when passed an error. It should be the only +// place where the application exits forcefully. +func (c *command) must(err error) { + if err != nil { + c.logger.Error( + fmt.Sprintf("Fatal error running command: %v", err), + "error", + err, + ) + os.Exit(1) + } +} diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 7138ba0..a199dac 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -80,6 +80,8 @@ type Config struct { DropboxAppSecret string `split_words:"true"` DropboxRemotePath string `split_words:"true"` DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"` + source string + additionalEnvVars map[string]string } type CompressionType string @@ -172,3 +174,40 @@ func (n *WholeNumber) Decode(v string) error { func (n *WholeNumber) Int() int { return int(*n) } + +type envVarLookup struct { + ok bool + key string + value string +} + +// applyEnv sets the values in `additionalEnvVars` as environment variables. +// It returns a function that reverts all values that have been set to its +// previous state. +func (c *Config) applyEnv() (func() error, error) { + lookups := []envVarLookup{} + + unset := func() error { + for _, lookup := range lookups { + if !lookup.ok { + if err := os.Unsetenv(lookup.key); err != nil { + return fmt.Errorf("(*Config).applyEnv: error unsetting env var %s: %w", lookup.key, err) + } + continue + } + if err := os.Setenv(lookup.key, lookup.value); err != nil { + return fmt.Errorf("(*Config).applyEnv: error setting back env var %s: %w", lookup.key, err) + } + } + return nil + } + + for key, value := range c.additionalEnvVars { + current, ok := os.LookupEnv(key) + lookups = append(lookups, envVarLookup{ok: ok, key: key, value: current}) + if err := os.Setenv(key, value); err != nil { + return unset, fmt.Errorf("(*Config).applyEnv: error setting env var: %w", err) + } + } + return unset, nil +} diff --git a/cmd/backup/config_provider.go b/cmd/backup/config_provider.go index 6c6a32a..b0a2755 100644 --- a/cmd/backup/config_provider.go +++ b/cmd/backup/config_provider.go @@ -12,9 +12,39 @@ import ( "github.com/offen/envconfig" ) +type configStrategy string + +const ( + configStrategyEnv configStrategy = "env" + configStrategyConfd configStrategy = "confd" +) + +// sourceConfiguration returns a list of config objects using the given +// strategy. It should be the single entrypoint for retrieving configuration +// for all consumers. +func sourceConfiguration(strategy configStrategy) ([]*Config, error) { + switch strategy { + case configStrategyEnv: + c, err := loadConfigFromEnvVars() + return []*Config{c}, err + case configStrategyConfd: + cs, err := loadConfigsFromEnvFiles("/etc/dockervolumebackup/conf.d") + if err != nil { + if os.IsNotExist(err) { + return sourceConfiguration(configStrategyEnv) + } + return nil, fmt.Errorf("sourceConfiguration: error loading config files: %w", err) + } + return cs, nil + default: + return nil, fmt.Errorf("sourceConfiguration: received unknown config strategy: %v", strategy) + } +} + // envProxy is a function that mimics os.LookupEnv but can read values from any other source type envProxy func(string) (string, bool) +// loadConfig creates a config object using the given lookup function func loadConfig(lookup envProxy) (*Config, error) { envconfig.Lookup = func(key string) (string, bool) { value, okValue := lookup(key) @@ -44,17 +74,16 @@ func loadConfig(lookup envProxy) (*Config, error) { return c, nil } -func loadEnvVars() (*Config, error) { - return loadConfig(os.LookupEnv) +func loadConfigFromEnvVars() (*Config, error) { + c, err := loadConfig(os.LookupEnv) + if err != nil { + return nil, fmt.Errorf("loadEnvVars: error loading config from environment: %w", err) + } + c.source = "from environment" + return c, nil } -type configFile struct { - name string - config *Config - additionalEnvVars map[string]string -} - -func loadEnvFiles(directory string) ([]configFile, error) { +func loadConfigsFromEnvFiles(directory string) ([]*Config, error) { items, err := os.ReadDir(directory) if err != nil { if os.IsNotExist(err) { @@ -63,7 +92,7 @@ func loadEnvFiles(directory string) ([]configFile, error) { return nil, fmt.Errorf("loadEnvFiles: failed to read files from env directory: %w", err) } - cs := []configFile{} + configs := []*Config{} for _, item := range items { if item.IsDir() { continue @@ -88,8 +117,10 @@ func loadEnvFiles(directory string) ([]configFile, error) { if err != nil { return nil, fmt.Errorf("loadEnvFiles: error loading config from file %s: %w", p, err) } - cs = append(cs, configFile{config: c, name: item.Name(), additionalEnvVars: envFile}) + c.source = item.Name() + c.additionalEnvVars = envFile + configs = append(configs, c) } - return cs, nil + return configs, nil } diff --git a/cmd/backup/copy_archive.go b/cmd/backup/copy_archive.go new file mode 100644 index 0000000..d111099 --- /dev/null +++ b/cmd/backup/copy_archive.go @@ -0,0 +1,41 @@ +// Copyright 2024 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package main + +import ( + "fmt" + "os" + "path" + + "golang.org/x/sync/errgroup" +) + +// copyArchive makes sure the backup file is copied to both local and remote locations +// as per the given configuration. +func (s *script) copyArchive() error { + _, name := path.Split(s.file) + if stat, err := os.Stat(s.file); err != nil { + return fmt.Errorf("copyArchive: unable to stat backup file: %w", err) + } else { + size := stat.Size() + s.stats.BackupFile = BackupFileStats{ + Size: uint64(size), + Name: name, + FullPath: s.file, + } + } + + eg := errgroup.Group{} + for _, backend := range s.storages { + b := backend + eg.Go(func() error { + return b.Copy(s.file) + }) + } + if err := eg.Wait(); err != nil { + return fmt.Errorf("copyArchive: error copying archive: %w", err) + } + + return nil +} diff --git a/cmd/backup/create_archive.go b/cmd/backup/create_archive.go new file mode 100644 index 0000000..de6d3f2 --- /dev/null +++ b/cmd/backup/create_archive.go @@ -0,0 +1,87 @@ +// Copyright 2024 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package main + +import ( + "fmt" + "io/fs" + "path/filepath" + + "github.com/otiai10/copy" +) + +// createArchive creates a tar archive of the configured backup location and +// saves it to disk. +func (s *script) createArchive() error { + backupSources := s.c.BackupSources + + if s.c.BackupFromSnapshot { + s.logger.Warn( + "Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.", + ) + s.logger.Warn( + "Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the documentation for an upgrade guide.", + ) + backupSources = filepath.Join("/tmp", s.c.BackupSources) + // copy before compressing guard against a situation where backup folder's content are still growing. + s.registerHook(hookLevelPlumbing, func(error) error { + if err := remove(backupSources); err != nil { + return fmt.Errorf("createArchive: error removing snapshot: %w", err) + } + s.logger.Info( + fmt.Sprintf("Removed snapshot `%s`.", backupSources), + ) + return nil + }) + if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{ + PreserveTimes: true, + PreserveOwner: true, + }); err != nil { + return fmt.Errorf("createArchive: error creating snapshot: %w", err) + } + s.logger.Info( + fmt.Sprintf("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources), + ) + } + + tarFile := s.file + s.registerHook(hookLevelPlumbing, func(error) error { + if err := remove(tarFile); err != nil { + return fmt.Errorf("createArchive: error removing tar file: %w", err) + } + s.logger.Info( + fmt.Sprintf("Removed tar file `%s`.", tarFile), + ) + return nil + }) + + backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources)) + if err != nil { + return fmt.Errorf("createArchive: error getting absolute path: %w", err) + } + + var filesEligibleForBackup []string + if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error { + if err != nil { + return err + } + + if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) { + return nil + } + filesEligibleForBackup = append(filesEligibleForBackup, path) + return nil + }); err != nil { + return fmt.Errorf("createArchive: error walking filesystem tree: %w", err) + } + + if err := createArchive(filesEligibleForBackup, backupSources, tarFile, s.c.BackupCompression.String(), s.c.GzipParallelism.Int()); err != nil { + return fmt.Errorf("createArchive: error compressing backup folder: %w", err) + } + + s.logger.Info( + fmt.Sprintf("Created backup of `%s` at `%s`.", backupSources, tarFile), + ) + return nil +} diff --git a/cmd/backup/cron.go b/cmd/backup/cron.go deleted file mode 100644 index 9416591..0000000 --- a/cmd/backup/cron.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2024 - Offen Authors -// SPDX-License-Identifier: MPL-2.0 - -package main - -import ( - "time" - - "github.com/robfig/cron/v3" -) - -// checkCronSchedule detects whether the given cron expression will actually -// ever be executed or not. -func checkCronSchedule(expression string) (ok bool) { - defer func() { - if err := recover(); err != nil { - ok = false - } - }() - sched, err := cron.ParseStandard(expression) - if err != nil { - ok = false - return - } - now := time.Now() - sched.Next(now) // panics when the cron would never run - ok = true - return -} diff --git a/cmd/backup/encrypt_archive.go b/cmd/backup/encrypt_archive.go new file mode 100644 index 0000000..4bc38dd --- /dev/null +++ b/cmd/backup/encrypt_archive.go @@ -0,0 +1,63 @@ +// Copyright 2024 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package main + +import ( + "fmt" + "io" + "os" + "path" + + openpgp "github.com/ProtonMail/go-crypto/openpgp/v2" +) + +// encryptArchive encrypts the backup file using PGP and the configured passphrase. +// In case no passphrase is given it returns early, leaving the backup file +// untouched. +func (s *script) encryptArchive() error { + if s.c.GpgPassphrase == "" { + return nil + } + + gpgFile := fmt.Sprintf("%s.gpg", s.file) + s.registerHook(hookLevelPlumbing, func(error) error { + if err := remove(gpgFile); err != nil { + return fmt.Errorf("encryptArchive: error removing gpg file: %w", err) + } + s.logger.Info( + fmt.Sprintf("Removed GPG file `%s`.", gpgFile), + ) + return nil + }) + + outFile, err := os.Create(gpgFile) + if err != nil { + return fmt.Errorf("encryptArchive: error opening out file: %w", err) + } + defer outFile.Close() + + _, name := path.Split(s.file) + dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{ + FileName: name, + }, nil) + if err != nil { + return fmt.Errorf("encryptArchive: error encrypting backup file: %w", err) + } + defer dst.Close() + + src, err := os.Open(s.file) + if err != nil { + return fmt.Errorf("encryptArchive: error opening backup file `%s`: %w", s.file, err) + } + + if _, err := io.Copy(dst, src); err != nil { + return fmt.Errorf("encryptArchive: error writing ciphertext to file: %w", err) + } + + s.file = gpgFile + s.logger.Info( + fmt.Sprintf("Encrypted backup using given passphrase, saving as `%s`.", s.file), + ) + return nil +} diff --git a/cmd/backup/exec.go b/cmd/backup/exec.go index 1fa0997..d001469 100644 --- a/cmd/backup/exec.go +++ b/cmd/backup/exec.go @@ -9,6 +9,7 @@ package main import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -190,13 +191,12 @@ func (s *script) withLabeledCommands(step lifecyclePhase, cb func() error) func( } return func() (err error) { if err = s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil { - err = fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err) + err = fmt.Errorf("(*script).withLabeledCommands: %s: error running pre commands: %w", step, err) return } defer func() { - derr := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step)) - if err == nil && derr != nil { - err = derr + if derr := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step)); derr != nil { + err = errors.Join(err, fmt.Errorf("(*script).withLabeledCommands: error running %s-post commands: %w", step, derr)) } }() err = cb() diff --git a/cmd/backup/main.go b/cmd/backup/main.go index fb23c79..956a8ee 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -5,230 +5,8 @@ package main import ( "flag" - "fmt" - "log/slog" - "os" - "os/signal" - "runtime" - "syscall" - - "github.com/robfig/cron/v3" ) -type command struct { - logger *slog.Logger -} - -func newCommand() *command { - return &command{ - logger: slog.New(slog.NewTextHandler(os.Stdout, nil)), - } -} - -func (c *command) must(err error) { - if err != nil { - c.logger.Error( - fmt.Sprintf("Fatal error running command: %v", err), - "error", - err, - ) - os.Exit(1) - } -} - -func runScript(c *Config, envVars map[string]string) (err error) { - defer func() { - if derr := recover(); derr != nil { - err = fmt.Errorf("runScript: unexpected panic running script: %v", err) - } - }() - - s, unlock, err := newScript(c, envVars) - if err != nil { - err = fmt.Errorf("runScript: error instantiating script: %w", err) - return - } - defer func() { - derr := unlock() - if err != nil { - err = fmt.Errorf("runScript: error releasing file lock: %w", derr) - } - }() - - runErr := func() (err error) { - scriptErr := func() error { - if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) { - restartContainersAndServices, err := s.stopContainersAndServices() - // The mechanism for restarting containers is not using hooks as it - // should happen as soon as possible (i.e. before uploading backups or - // similar). - defer func() { - derr := restartContainersAndServices() - if err == nil { - err = derr - } - }() - if err != nil { - return - } - err = s.createArchive() - return - })(); err != nil { - return err - } - - if err := s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)(); err != nil { - return err - } - if err := s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)(); err != nil { - return err - } - if err := s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)(); err != nil { - return err - } - return nil - }() - - if hookErr := s.runHooks(scriptErr); hookErr != nil { - if scriptErr != nil { - return fmt.Errorf( - "runScript: error %w executing the script followed by %w calling the registered hooks", - scriptErr, - hookErr, - ) - } - return fmt.Errorf( - "runScript: the script ran successfully, but an error occurred calling the registered hooks: %w", - hookErr, - ) - } - if scriptErr != nil { - return fmt.Errorf("runScript: error running script: %w", scriptErr) - } - return nil - }() - - if runErr != nil { - s.logger.Error( - fmt.Sprintf("Script run failed: %v", runErr), "error", runErr, - ) - } - return runErr -} - -func (c *command) runInForeground(profileCronExpression string) error { - cr := cron.New( - cron.WithParser( - cron.NewParser( - cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor, - ), - ), - ) - - addJob := func(config *Config, name string, envVars map[string]string) error { - if _, err := cr.AddFunc(config.BackupCronExpression, func() { - c.logger.Info( - fmt.Sprintf( - "Now running script on schedule %s", - config.BackupCronExpression, - ), - ) - - if err := runScript(config, envVars); err != nil { - c.logger.Error( - fmt.Sprintf( - "Unexpected error running schedule %s: %v", - config.BackupCronExpression, - err, - ), - "error", - err, - ) - } - }); err != nil { - return fmt.Errorf("addJob: error adding schedule %s: %w", config.BackupCronExpression, err) - } - - c.logger.Info(fmt.Sprintf("Successfully scheduled backup %s with expression %s", name, config.BackupCronExpression)) - if ok := checkCronSchedule(config.BackupCronExpression); !ok { - c.logger.Warn( - fmt.Sprintf("Scheduled cron expression %s will never run, is this intentional?", config.BackupCronExpression), - ) - } - - return nil - } - - cs, err := loadEnvFiles("/etc/dockervolumebackup/conf.d") - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("runInForeground: could not load config from environment files: %w", err) - } - - c, err := loadEnvVars() - if err != nil { - return fmt.Errorf("runInForeground: could not load config from environment variables: %w", err) - } else { - err = addJob(c, "from environment", nil) - if err != nil { - return fmt.Errorf("runInForeground: error adding job from env: %w", err) - } - } - } else { - c.logger.Info("/etc/dockervolumebackup/conf.d was found, using configuration files from this directory.") - for _, config := range cs { - err = addJob(config.config, config.name, config.additionalEnvVars) - if err != nil { - return fmt.Errorf("runInForeground: error adding jobs from conf files: %w", err) - } - } - } - - if profileCronExpression != "" { - if _, err := cr.AddFunc(profileCronExpression, func() { - memStats := runtime.MemStats{} - runtime.ReadMemStats(&memStats) - c.logger.Info( - "Collecting runtime information", - "num_goroutines", - runtime.NumGoroutine(), - "memory_heap_alloc", - formatBytes(memStats.HeapAlloc, false), - "memory_heap_inuse", - formatBytes(memStats.HeapInuse, false), - "memory_heap_sys", - formatBytes(memStats.HeapSys, false), - "memory_heap_objects", - memStats.HeapObjects, - ) - }); err != nil { - return fmt.Errorf("runInForeground: error adding profiling job: %w", err) - } - } - - var quit = make(chan os.Signal, 1) - signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT) - cr.Start() - <-quit - ctx := cr.Stop() - <-ctx.Done() - - return nil -} - -func (c *command) runAsCommand() error { - config, err := loadEnvVars() - if err != nil { - return fmt.Errorf("runAsCommand: error loading env vars: %w", err) - } - err = runScript(config, nil) - if err != nil { - return fmt.Errorf("runAsCommand: error running script: %w", err) - } - - return nil -} - func main() { foreground := flag.Bool("foreground", false, "run the tool in the foreground") profile := flag.String("profile", "", "collect runtime metrics and log them periodically on the given cron expression") @@ -236,7 +14,10 @@ func main() { c := newCommand() if *foreground { - c.must(c.runInForeground(*profile)) + opts := foregroundOpts{ + profileCronExpression: *profile, + } + c.must(c.runInForeground(opts)) } else { c.must(c.runAsCommand()) } diff --git a/cmd/backup/profile.go b/cmd/backup/profile.go new file mode 100644 index 0000000..5666503 --- /dev/null +++ b/cmd/backup/profile.go @@ -0,0 +1,24 @@ +// Copyright 2024 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package main + +import "runtime" + +func (c *command) profile() { + memStats := runtime.MemStats{} + runtime.ReadMemStats(&memStats) + c.logger.Info( + "Collecting runtime information", + "num_goroutines", + runtime.NumGoroutine(), + "memory_heap_alloc", + formatBytes(memStats.HeapAlloc, false), + "memory_heap_inuse", + formatBytes(memStats.HeapInuse, false), + "memory_heap_sys", + formatBytes(memStats.HeapSys, false), + "memory_heap_objects", + memStats.HeapObjects, + ) +} diff --git a/cmd/backup/prune_backups.go b/cmd/backup/prune_backups.go new file mode 100644 index 0000000..cde0673 --- /dev/null +++ b/cmd/backup/prune_backups.go @@ -0,0 +1,65 @@ +// Copyright 2024 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package main + +import ( + "fmt" + "slices" + "strings" + "time" + + "golang.org/x/sync/errgroup" +) + +// pruneBackups rotates away backups from local and remote storages using +// the given configuration. In case the given configuration would delete all +// backups, it does nothing instead and logs a warning. +func (s *script) pruneBackups() error { + if s.c.BackupRetentionDays < 0 { + return nil + } + + deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway) + + eg := errgroup.Group{} + for _, backend := range s.storages { + b := backend + eg.Go(func() error { + if skipPrune(b.Name(), s.c.BackupSkipBackendsFromPrune) { + s.logger.Info( + fmt.Sprintf("Skipping pruning for backend `%s`.", b.Name()), + ) + return nil + } + stats, err := b.Prune(deadline, s.c.BackupPruningPrefix) + if err != nil { + return err + } + s.stats.Lock() + s.stats.Storages[b.Name()] = StorageStats{ + Total: stats.Total, + Pruned: stats.Pruned, + } + s.stats.Unlock() + return nil + }) + } + + if err := eg.Wait(); err != nil { + return fmt.Errorf("pruneBackups: error pruning backups: %w", err) + } + + return nil +} + +// skipPrune returns true if the given backend name is contained in the +// list of skipped backends. +func skipPrune(name string, skippedBackends []string) bool { + return slices.ContainsFunc( + skippedBackends, + func(b string) bool { + return strings.EqualFold(b, name) // ignore case on both sides + }, + ) +} diff --git a/cmd/backup/run_script.go b/cmd/backup/run_script.go new file mode 100644 index 0000000..e734f10 --- /dev/null +++ b/cmd/backup/run_script.go @@ -0,0 +1,101 @@ +// Copyright 2024 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package main + +import ( + "errors" + "fmt" +) + +// runScript instantiates a new script object and orchestrates a backup run. +// To ensure it runs mutually exclusive a global file lock is acquired before +// it starts running. Any panic within the script will be recovered and returned +// as an error. +func runScript(c *Config) (err error) { + defer func() { + if derr := recover(); derr != nil { + err = fmt.Errorf("runScript: unexpected panic running script: %v", derr) + } + }() + + s := newScript(c) + + unlock, lockErr := s.lock("/var/lock/dockervolumebackup.lock") + if lockErr != nil { + err = fmt.Errorf("runScript: error acquiring file lock: %w", lockErr) + return + } + defer func() { + if derr := unlock(); derr != nil { + err = errors.Join(err, fmt.Errorf("runScript: error releasing file lock: %w", derr)) + } + }() + + unset, err := s.c.applyEnv() + if err != nil { + return fmt.Errorf("runScript: error applying env: %w", err) + } + defer func() { + if derr := unset(); derr != nil { + err = errors.Join(err, fmt.Errorf("runScript: error unsetting environment variables: %w", derr)) + } + }() + + if initErr := s.init(); initErr != nil { + err = fmt.Errorf("runScript: error instantiating script: %w", initErr) + return + } + + return func() (err error) { + scriptErr := func() error { + if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) { + restartContainersAndServices, err := s.stopContainersAndServices() + // The mechanism for restarting containers is not using hooks as it + // should happen as soon as possible (i.e. before uploading backups or + // similar). + defer func() { + if derr := restartContainersAndServices(); derr != nil { + err = errors.Join(err, fmt.Errorf("runScript: error restarting containers and services: %w", derr)) + } + }() + if err != nil { + return + } + err = s.createArchive() + return + })(); err != nil { + return err + } + + if err := s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)(); err != nil { + return err + } + if err := s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)(); err != nil { + return err + } + if err := s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)(); err != nil { + return err + } + return nil + }() + + if hookErr := s.runHooks(scriptErr); hookErr != nil { + if scriptErr != nil { + return fmt.Errorf( + "runScript: error %w executing the script followed by %w calling the registered hooks", + scriptErr, + hookErr, + ) + } + return fmt.Errorf( + "runScript: the script ran successfully, but an error occurred calling the registered hooks: %w", + hookErr, + ) + } + if scriptErr != nil { + return fmt.Errorf("runScript: error running script: %w", scriptErr) + } + return nil + }() +} diff --git a/cmd/backup/script.go b/cmd/backup/script.go index a4c4a9b..492dcb3 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -6,14 +6,9 @@ package main import ( "bytes" "fmt" - "io" - "io/fs" "log/slog" "os" "path" - "path/filepath" - "slices" - "strings" "text/template" "time" @@ -25,13 +20,10 @@ import ( "github.com/offen/docker-volume-backup/internal/storage/ssh" "github.com/offen/docker-volume-backup/internal/storage/webdav" - openpgp "github.com/ProtonMail/go-crypto/openpgp/v2" "github.com/containrrr/shoutrrr" "github.com/containrrr/shoutrrr/pkg/router" "github.com/docker/docker/client" "github.com/leekchan/timeutil" - "github.com/otiai10/copy" - "golang.org/x/sync/errgroup" ) // script holds all the stateful information required to orchestrate a @@ -57,9 +49,9 @@ type script struct { // remote resources like the Docker engine or remote storage locations. All // reading from env vars or other configuration sources is expected to happen // in this method. -func newScript(c *Config, envVars map[string]string) (*script, func() error, error) { +func newScript(c *Config) *script { stdOut, logBuffer := buffer(os.Stdout) - s := &script{ + return &script{ c: c, logger: slog.New(slog.NewTextHandler(stdOut, nil)), stats: &Stats{ @@ -75,30 +67,9 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err }, }, } +} - unlock, err := s.lock("/var/lock/dockervolumebackup.lock") - if err != nil { - return nil, noop, fmt.Errorf("runScript: error acquiring file lock: %w", err) - } - - for key, value := range envVars { - currentVal, currentOk := os.LookupEnv(key) - defer func(currentKey, currentVal string, currentOk bool) { - if !currentOk { - _ = os.Unsetenv(currentKey) - } else { - _ = os.Setenv(currentKey, currentVal) - } - }(key, currentVal, currentOk) - - if err := os.Setenv(key, value); err != nil { - return nil, unlock, fmt.Errorf( - "Unexpected error overloading environment %s: %w", - s.c.BackupCronExpression, - err, - ) - } - } +func (s *script) init() error { s.registerHook(hookLevelPlumbing, func(error) error { s.stats.EndTime = time.Now() s.stats.TookTime = s.stats.EndTime.Sub(s.stats.StartTime) @@ -109,14 +80,14 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err tmplFileName, tErr := template.New("extension").Parse(s.file) if tErr != nil { - return nil, unlock, fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr) + return fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr) } var bf bytes.Buffer if tErr := tmplFileName.Execute(&bf, map[string]string{ "Extension": fmt.Sprintf("tar.%s", s.c.BackupCompression), }); tErr != nil { - return nil, unlock, fmt.Errorf("newScript: error executing backup file extension template: %w", tErr) + return fmt.Errorf("newScript: error executing backup file extension template: %w", tErr) } s.file = bf.String() @@ -127,12 +98,12 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err } s.file = timeutil.Strftime(&s.stats.StartTime, s.file) - _, err = os.Stat("/var/run/docker.sock") + _, err := os.Stat("/var/run/docker.sock") _, dockerHostSet := os.LookupEnv("DOCKER_HOST") if !os.IsNotExist(err) || dockerHostSet { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { - return nil, unlock, fmt.Errorf("newScript: failed to create docker client") + return fmt.Errorf("newScript: failed to create docker client") } s.cli = cli s.registerHook(hookLevelPlumbing, func(err error) error { @@ -170,7 +141,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err } s3Backend, err := s3.NewStorageBackend(s3Config, logFunc) if err != nil { - return nil, unlock, fmt.Errorf("newScript: error creating s3 storage backend: %w", err) + return fmt.Errorf("newScript: error creating s3 storage backend: %w", err) } s.storages = append(s.storages, s3Backend) } @@ -185,7 +156,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err } webdavBackend, err := webdav.NewStorageBackend(webDavConfig, logFunc) if err != nil { - return nil, unlock, fmt.Errorf("newScript: error creating webdav storage backend: %w", err) + return fmt.Errorf("newScript: error creating webdav storage backend: %w", err) } s.storages = append(s.storages, webdavBackend) } @@ -202,7 +173,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err } sshBackend, err := ssh.NewStorageBackend(sshConfig, logFunc) if err != nil { - return nil, unlock, fmt.Errorf("newScript: error creating ssh storage backend: %w", err) + return fmt.Errorf("newScript: error creating ssh storage backend: %w", err) } s.storages = append(s.storages, sshBackend) } @@ -226,7 +197,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err } azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) if err != nil { - return nil, unlock, fmt.Errorf("newScript: error creating azure storage backend: %w", err) + return fmt.Errorf("newScript: error creating azure storage backend: %w", err) } s.storages = append(s.storages, azureBackend) } @@ -243,7 +214,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err } dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc) if err != nil { - return nil, unlock, fmt.Errorf("newScript: error creating dropbox storage backend: %w", err) + return fmt.Errorf("newScript: error creating dropbox storage backend: %w", err) } s.storages = append(s.storages, dropboxBackend) } @@ -269,14 +240,14 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err hookLevel, ok := hookLevels[s.c.NotificationLevel] if !ok { - return nil, unlock, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel) + return fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel) } s.hookLevel = hookLevel if len(s.c.NotificationURLs) > 0 { sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...) if senderErr != nil { - return nil, unlock, fmt.Errorf("newScript: error creating sender: %w", senderErr) + return fmt.Errorf("newScript: error creating sender: %w", senderErr) } s.sender = sender @@ -284,13 +255,13 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err tmpl.Funcs(templateHelpers) tmpl, err = tmpl.Parse(defaultNotifications) if err != nil { - return nil, unlock, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err) + return fmt.Errorf("newScript: unable to parse default notifications templates: %w", err) } if fi, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil && fi.IsDir() { tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*") if err != nil { - return nil, unlock, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err) + return fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err) } } s.template = tmpl @@ -311,211 +282,5 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err }) } - return s, unlock, nil -} - -// createArchive creates a tar archive of the configured backup location and -// saves it to disk. -func (s *script) createArchive() error { - backupSources := s.c.BackupSources - - if s.c.BackupFromSnapshot { - s.logger.Warn( - "Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.", - ) - s.logger.Warn( - "Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the documentation for an upgrade guide.", - ) - backupSources = filepath.Join("/tmp", s.c.BackupSources) - // copy before compressing guard against a situation where backup folder's content are still growing. - s.registerHook(hookLevelPlumbing, func(error) error { - if err := remove(backupSources); err != nil { - return fmt.Errorf("createArchive: error removing snapshot: %w", err) - } - s.logger.Info( - fmt.Sprintf("Removed snapshot `%s`.", backupSources), - ) - return nil - }) - if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{ - PreserveTimes: true, - PreserveOwner: true, - }); err != nil { - return fmt.Errorf("createArchive: error creating snapshot: %w", err) - } - s.logger.Info( - fmt.Sprintf("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources), - ) - } - - tarFile := s.file - s.registerHook(hookLevelPlumbing, func(error) error { - if err := remove(tarFile); err != nil { - return fmt.Errorf("createArchive: error removing tar file: %w", err) - } - s.logger.Info( - fmt.Sprintf("Removed tar file `%s`.", tarFile), - ) - return nil - }) - - backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources)) - if err != nil { - return fmt.Errorf("createArchive: error getting absolute path: %w", err) - } - - var filesEligibleForBackup []string - if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error { - if err != nil { - return err - } - - if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) { - return nil - } - filesEligibleForBackup = append(filesEligibleForBackup, path) - return nil - }); err != nil { - return fmt.Errorf("createArchive: error walking filesystem tree: %w", err) - } - - if err := createArchive(filesEligibleForBackup, backupSources, tarFile, s.c.BackupCompression.String(), s.c.GzipParallelism.Int()); err != nil { - return fmt.Errorf("createArchive: error compressing backup folder: %w", err) - } - - s.logger.Info( - fmt.Sprintf("Created backup of `%s` at `%s`.", backupSources, tarFile), - ) return nil } - -// encryptArchive encrypts the backup file using PGP and the configured passphrase. -// In case no passphrase is given it returns early, leaving the backup file -// untouched. -func (s *script) encryptArchive() error { - if s.c.GpgPassphrase == "" { - return nil - } - - gpgFile := fmt.Sprintf("%s.gpg", s.file) - s.registerHook(hookLevelPlumbing, func(error) error { - if err := remove(gpgFile); err != nil { - return fmt.Errorf("encryptArchive: error removing gpg file: %w", err) - } - s.logger.Info( - fmt.Sprintf("Removed GPG file `%s`.", gpgFile), - ) - return nil - }) - - outFile, err := os.Create(gpgFile) - if err != nil { - return fmt.Errorf("encryptArchive: error opening out file: %w", err) - } - defer outFile.Close() - - _, name := path.Split(s.file) - dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{ - FileName: name, - }, nil) - if err != nil { - return fmt.Errorf("encryptArchive: error encrypting backup file: %w", err) - } - defer dst.Close() - - src, err := os.Open(s.file) - if err != nil { - return fmt.Errorf("encryptArchive: error opening backup file `%s`: %w", s.file, err) - } - - if _, err := io.Copy(dst, src); err != nil { - return fmt.Errorf("encryptArchive: error writing ciphertext to file: %w", err) - } - - s.file = gpgFile - s.logger.Info( - fmt.Sprintf("Encrypted backup using given passphrase, saving as `%s`.", s.file), - ) - return nil -} - -// copyArchive makes sure the backup file is copied to both local and remote locations -// as per the given configuration. -func (s *script) copyArchive() error { - _, name := path.Split(s.file) - if stat, err := os.Stat(s.file); err != nil { - return fmt.Errorf("copyArchive: unable to stat backup file: %w", err) - } else { - size := stat.Size() - s.stats.BackupFile = BackupFileStats{ - Size: uint64(size), - Name: name, - FullPath: s.file, - } - } - - eg := errgroup.Group{} - for _, backend := range s.storages { - b := backend - eg.Go(func() error { - return b.Copy(s.file) - }) - } - if err := eg.Wait(); err != nil { - return fmt.Errorf("copyArchive: error copying archive: %w", err) - } - - return nil -} - -// pruneBackups rotates away backups from local and remote storages using -// the given configuration. In case the given configuration would delete all -// backups, it does nothing instead and logs a warning. -func (s *script) pruneBackups() error { - if s.c.BackupRetentionDays < 0 { - return nil - } - - deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway) - - eg := errgroup.Group{} - for _, backend := range s.storages { - b := backend - eg.Go(func() error { - if skipPrune(b.Name(), s.c.BackupSkipBackendsFromPrune) { - s.logger.Info( - fmt.Sprintf("Skipping pruning for backend `%s`.", b.Name()), - ) - return nil - } - stats, err := b.Prune(deadline, s.c.BackupPruningPrefix) - if err != nil { - return err - } - s.stats.Lock() - s.stats.Storages[b.Name()] = StorageStats{ - Total: stats.Total, - Pruned: stats.Pruned, - } - s.stats.Unlock() - return nil - }) - } - - if err := eg.Wait(); err != nil { - return fmt.Errorf("pruneBackups: error pruning backups: %w", err) - } - - return nil -} - -// skipPrune returns true if the given backend name is contained in the -// list of skipped backends. -func skipPrune(name string, skippedBackends []string) bool { - return slices.ContainsFunc( - skippedBackends, - func(b string) bool { - return strings.EqualFold(b, name) // ignore case on both sides - }, - ) -} diff --git a/cmd/backup/util.go b/cmd/backup/util.go index e13da86..814b961 100644 --- a/cmd/backup/util.go +++ b/cmd/backup/util.go @@ -9,6 +9,9 @@ import ( "io" "os" "sync" + "time" + + "github.com/robfig/cron/v3" ) var noop = func() error { return nil } @@ -79,3 +82,22 @@ func (c *concurrentSlice[T]) append(v T) { func (c *concurrentSlice[T]) value() []T { return c.val } + +// checkCronSchedule detects whether the given cron expression will actually +// ever be executed or not. +func checkCronSchedule(expression string) (ok bool) { + defer func() { + if err := recover(); err != nil { + ok = false + } + }() + sched, err := cron.ParseStandard(expression) + if err != nil { + ok = false + return + } + now := time.Now() + sched.Next(now) // panics when the cron would never run + ok = true + return +}