Refactor handling of runtime configuration to prepare for reloading

cmd/backup/command.go

@@ -0,0 +1,155 @@
+// Copyright 2024 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/robfig/cron/v3"
+)
+
+type command struct {
+	logger    *slog.Logger
+	schedules []cron.EntryID
+	cr        *cron.Cron
+	reload    chan struct{}
+}
+
+func newCommand() *command {
+	return &command{
+		logger: slog.New(slog.NewTextHandler(os.Stdout, nil)),
+	}
+}
+
+// runAsCommand executes a backup run for each configuration that is available
+// and then returns
+func (c *command) runAsCommand() error {
+	configurations, err := sourceConfiguration(configStrategyEnv)
+	if err != nil {
+		return fmt.Errorf("runAsCommand: error loading env vars: %w", err)
+	}
+
+	for _, config := range configurations {
+		if err := runScript(config); err != nil {
+			return fmt.Errorf("runAsCommand: error running script: %w", err)
+		}
+	}
+
+	return nil
+}
+
+type foregroundOpts struct {
+	profileCronExpression string
+}
+
+// runInForeground starts the program as a long running process, scheduling
+// a job for each configuration that is available.
+func (c *command) runInForeground(opts foregroundOpts) error {
+	c.cr = cron.New(
+		cron.WithParser(
+			cron.NewParser(
+				cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
+			),
+		),
+	)
+
+	if err := c.schedule(configStrategyConfd); err != nil {
+		return fmt.Errorf("runInForeground: error scheduling: %w", err)
+	}
+
+	if opts.profileCronExpression != "" {
+		if _, err := c.cr.AddFunc(opts.profileCronExpression, c.profile); err != nil {
+			return fmt.Errorf("runInForeground: error adding profiling job: %w", err)
+		}
+	}
+
+	var quit = make(chan os.Signal, 1)
+	c.reload = make(chan struct{}, 1)
+	signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT)
+	c.cr.Start()
+
+	for {
+		select {
+		case <-quit:
+			ctx := c.cr.Stop()
+			<-ctx.Done()
+			return nil
+		case <-c.reload:
+			if err := c.schedule(configStrategyConfd); err != nil {
+				return fmt.Errorf("runInForeground: error reloading configuration: %w", err)
+			}
+		}
+	}
+}
+
+// schedule wipes all existing schedules and enqueues all schedules available
+// using the given configuration strategy
+func (c *command) schedule(strategy configStrategy) error {
+	for _, id := range c.schedules {
+		c.cr.Remove(id)
+	}
+
+	configurations, err := sourceConfiguration(strategy)
+	if err != nil {
+		return fmt.Errorf("schedule: error sourcing configuration: %w", err)
+	}
+
+	for _, cfg := range configurations {
+		config := cfg
+		id, err := c.cr.AddFunc(config.BackupCronExpression, func() {
+			c.logger.Info(
+				fmt.Sprintf(
+					"Now running script on schedule %s",
+					config.BackupCronExpression,
+				),
+			)
+
+			if err := runScript(config); err != nil {
+				c.logger.Error(
+					fmt.Sprintf(
+						"Unexpected error running schedule %s: %v",
+						config.BackupCronExpression,
+						err,
+					),
+					"error",
+					err,
+				)
+			}
+		})
+
+		if err != nil {
+			return fmt.Errorf("addJob: error adding schedule %s: %w", config.BackupCronExpression, err)
+		}
+		c.logger.Info(fmt.Sprintf("Successfully scheduled backup %s with expression %s", config.source, config.BackupCronExpression))
+		if ok := checkCronSchedule(config.BackupCronExpression); !ok {
+			c.logger.Warn(
+				fmt.Sprintf("Scheduled cron expression %s will never run, is this intentional?", config.BackupCronExpression),
+			)
+
+			if err != nil {
+				return fmt.Errorf("schedule: error scheduling: %w", err)
+			}
+			c.schedules = append(c.schedules, id)
+		}
+	}
+
+	return nil
+}
+
+// must exits the program when passed an error. It should be the only
+// place where the application exits forcefully.
+func (c *command) must(err error) {
+	if err != nil {
+		c.logger.Error(
+			fmt.Sprintf("Fatal error running command: %v", err),
+			"error",
+			err,
+		)
+		os.Exit(1)
+	}
+}

cmd/backup/config.go

@@ -80,6 +80,8 @@ type Config struct {
 	DropboxAppSecret              string          `split_words:"true"`
 	DropboxRemotePath             string          `split_words:"true"`
 	DropboxConcurrencyLevel       NaturalNumber   `split_words:"true" default:"6"`
+	source                        string
+	additionalEnvVars             map[string]string
 }
 
 type CompressionType string
@@ -172,3 +174,40 @@ func (n *WholeNumber) Decode(v string) error {
 func (n *WholeNumber) Int() int {
 	return int(*n)
 }
+
+type envVarLookup struct {
+	ok    bool
+	key   string
+	value string
+}
+
+// applyEnv sets the values in `additionalEnvVars` as environment variables.
+// It returns a function that reverts all values that have been set to its
+// previous state.
+func (c *Config) applyEnv() (func() error, error) {
+	lookups := []envVarLookup{}
+
+	unset := func() error {
+		for _, lookup := range lookups {
+			if !lookup.ok {
+				if err := os.Unsetenv(lookup.key); err != nil {
+					return fmt.Errorf("(*Config).applyEnv: error unsetting env var %s: %w", lookup.key, err)
+				}
+				continue
+			}
+			if err := os.Setenv(lookup.key, lookup.value); err != nil {
+				return fmt.Errorf("(*Config).applyEnv: error setting back env var %s: %w", lookup.key, err)
+			}
+		}
+		return nil
+	}
+
+	for key, value := range c.additionalEnvVars {
+		current, ok := os.LookupEnv(key)
+		lookups = append(lookups, envVarLookup{ok: ok, key: key, value: current})
+		if err := os.Setenv(key, value); err != nil {
+			return unset, fmt.Errorf("(*Config).applyEnv: error setting env var: %w", err)
+		}
+	}
+	return unset, nil
+}

cmd/backup/config_provider.go

@@ -12,9 +12,39 @@ import (
 	"github.com/offen/envconfig"
 )
 
+type configStrategy string
+
+const (
+	configStrategyEnv   configStrategy = "env"
+	configStrategyConfd configStrategy = "confd"
+)
+
+// sourceConfiguration returns a list of config objects using the given
+// strategy. It should be the single entrypoint for retrieving configuration
+// for all consumers.
+func sourceConfiguration(strategy configStrategy) ([]*Config, error) {
+	switch strategy {
+	case configStrategyEnv:
+		c, err := loadConfigFromEnvVars()
+		return []*Config{c}, err
+	case configStrategyConfd:
+		cs, err := loadConfigsFromEnvFiles("/etc/dockervolumebackup/conf.d")
+		if err != nil {
+			if os.IsNotExist(err) {
+				return sourceConfiguration(configStrategyEnv)
+			}
+			return nil, fmt.Errorf("sourceConfiguration: error loading config files: %w", err)
+		}
+		return cs, nil
+	default:
+		return nil, fmt.Errorf("sourceConfiguration: received unknown config strategy: %v", strategy)
+	}
+}
+
 // envProxy is a function that mimics os.LookupEnv but can read values from any other source
 type envProxy func(string) (string, bool)
 
+// loadConfig creates a config object using the given lookup function
 func loadConfig(lookup envProxy) (*Config, error) {
 	envconfig.Lookup = func(key string) (string, bool) {
 		value, okValue := lookup(key)
@@ -44,17 +74,16 @@ func loadConfig(lookup envProxy) (*Config, error) {
 	return c, nil
 }
 
-func loadEnvVars() (*Config, error) {
+func loadConfigFromEnvVars() (*Config, error) {
-	return loadConfig(os.LookupEnv)
+	c, err := loadConfig(os.LookupEnv)
+	if err != nil {
+		return nil, fmt.Errorf("loadEnvVars: error loading config from environment: %w", err)
+	}
+	c.source = "from environment"
+	return c, nil
 }
 
-type configFile struct {
+func loadConfigsFromEnvFiles(directory string) ([]*Config, error) {
-	name              string
-	config            *Config
-	additionalEnvVars map[string]string
-}
-
-func loadEnvFiles(directory string) ([]configFile, error) {
 	items, err := os.ReadDir(directory)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -63,7 +92,7 @@ func loadEnvFiles(directory string) ([]configFile, error) {
 		return nil, fmt.Errorf("loadEnvFiles: failed to read files from env directory: %w", err)
 	}
 
-	cs := []configFile{}
+	configs := []*Config{}
 	for _, item := range items {
 		if item.IsDir() {
 			continue
@@ -88,8 +117,10 @@ func loadEnvFiles(directory string) ([]configFile, error) {
 		if err != nil {
 			return nil, fmt.Errorf("loadEnvFiles: error loading config from file %s: %w", p, err)
 		}
-		cs = append(cs, configFile{config: c, name: item.Name(), additionalEnvVars: envFile})
+		c.source = item.Name()
+		c.additionalEnvVars = envFile
+		configs = append(configs, c)
 	}
 
-	return cs, nil
+	return configs, nil
 }

cmd/backup/copy_archive.go

@@ -0,0 +1,41 @@
+// Copyright 2024 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// copyArchive makes sure the backup file is copied to both local and remote locations
+// as per the given configuration.
+func (s *script) copyArchive() error {
+	_, name := path.Split(s.file)
+	if stat, err := os.Stat(s.file); err != nil {
+		return fmt.Errorf("copyArchive: unable to stat backup file: %w", err)
+	} else {
+		size := stat.Size()
+		s.stats.BackupFile = BackupFileStats{
+			Size:     uint64(size),
+			Name:     name,
+			FullPath: s.file,
+		}
+	}
+
+	eg := errgroup.Group{}
+	for _, backend := range s.storages {
+		b := backend
+		eg.Go(func() error {
+			return b.Copy(s.file)
+		})
+	}
+	if err := eg.Wait(); err != nil {
+		return fmt.Errorf("copyArchive: error copying archive: %w", err)
+	}
+
+	return nil
+}

cmd/backup/create_archive.go

@@ -0,0 +1,87 @@
+// Copyright 2024 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+
+	"github.com/otiai10/copy"
+)
+
+// createArchive creates a tar archive of the configured backup location and
+// saves it to disk.
+func (s *script) createArchive() error {
+	backupSources := s.c.BackupSources
+
+	if s.c.BackupFromSnapshot {
+		s.logger.Warn(
+			"Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.",
+		)
+		s.logger.Warn(
+			"Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the documentation for an upgrade guide.",
+		)
+		backupSources = filepath.Join("/tmp", s.c.BackupSources)
+		// copy before compressing guard against a situation where backup folder's content are still growing.
+		s.registerHook(hookLevelPlumbing, func(error) error {
+			if err := remove(backupSources); err != nil {
+				return fmt.Errorf("createArchive: error removing snapshot: %w", err)
+			}
+			s.logger.Info(
+				fmt.Sprintf("Removed snapshot `%s`.", backupSources),
+			)
+			return nil
+		})
+		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
+			PreserveTimes: true,
+			PreserveOwner: true,
+		}); err != nil {
+			return fmt.Errorf("createArchive: error creating snapshot: %w", err)
+		}
+		s.logger.Info(
+			fmt.Sprintf("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources),
+		)
+	}
+
+	tarFile := s.file
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		if err := remove(tarFile); err != nil {
+			return fmt.Errorf("createArchive: error removing tar file: %w", err)
+		}
+		s.logger.Info(
+			fmt.Sprintf("Removed tar file `%s`.", tarFile),
+		)
+		return nil
+	})
+
+	backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources))
+	if err != nil {
+		return fmt.Errorf("createArchive: error getting absolute path: %w", err)
+	}
+
+	var filesEligibleForBackup []string
+	if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) {
+			return nil
+		}
+		filesEligibleForBackup = append(filesEligibleForBackup, path)
+		return nil
+	}); err != nil {
+		return fmt.Errorf("createArchive: error walking filesystem tree: %w", err)
+	}
+
+	if err := createArchive(filesEligibleForBackup, backupSources, tarFile, s.c.BackupCompression.String(), s.c.GzipParallelism.Int()); err != nil {
+		return fmt.Errorf("createArchive: error compressing backup folder: %w", err)
+	}
+
+	s.logger.Info(
+		fmt.Sprintf("Created backup of `%s` at `%s`.", backupSources, tarFile),
+	)
+	return nil
+}

cmd/backup/cron.go

@@ -1,29 +0,0 @@
-// Copyright 2024 - Offen Authors <hioffen@posteo.de>
-// SPDX-License-Identifier: MPL-2.0
-
-package main
-
-import (
-	"time"
-
-	"github.com/robfig/cron/v3"
-)
-
-// checkCronSchedule detects whether the given cron expression will actually
-// ever be executed or not.
-func checkCronSchedule(expression string) (ok bool) {
-	defer func() {
-		if err := recover(); err != nil {
-			ok = false
-		}
-	}()
-	sched, err := cron.ParseStandard(expression)
-	if err != nil {
-		ok = false
-		return
-	}
-	now := time.Now()
-	sched.Next(now) // panics when the cron would never run
-	ok = true
-	return
-}

cmd/backup/encrypt_archive.go

@@ -0,0 +1,63 @@
+// Copyright 2024 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+
+	openpgp "github.com/ProtonMail/go-crypto/openpgp/v2"
+)
+
+// encryptArchive encrypts the backup file using PGP and the configured passphrase.
+// In case no passphrase is given it returns early, leaving the backup file
+// untouched.
+func (s *script) encryptArchive() error {
+	if s.c.GpgPassphrase == "" {
+		return nil
+	}
+
+	gpgFile := fmt.Sprintf("%s.gpg", s.file)
+	s.registerHook(hookLevelPlumbing, func(error) error {
+		if err := remove(gpgFile); err != nil {
+			return fmt.Errorf("encryptArchive: error removing gpg file: %w", err)
+		}
+		s.logger.Info(
+			fmt.Sprintf("Removed GPG file `%s`.", gpgFile),
+		)
+		return nil
+	})
+
+	outFile, err := os.Create(gpgFile)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error opening out file: %w", err)
+	}
+	defer outFile.Close()
+
+	_, name := path.Split(s.file)
+	dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
+		FileName: name,
+	}, nil)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error encrypting backup file: %w", err)
+	}
+	defer dst.Close()
+
+	src, err := os.Open(s.file)
+	if err != nil {
+		return fmt.Errorf("encryptArchive: error opening backup file `%s`: %w", s.file, err)
+	}
+
+	if _, err := io.Copy(dst, src); err != nil {
+		return fmt.Errorf("encryptArchive: error writing ciphertext to file: %w", err)
+	}
+
+	s.file = gpgFile
+	s.logger.Info(
+		fmt.Sprintf("Encrypted backup using given passphrase, saving as `%s`.", s.file),
+	)
+	return nil
+}

cmd/backup/exec.go

@@ -9,6 +9,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -190,13 +191,12 @@ func (s *script) withLabeledCommands(step lifecyclePhase, cb func() error) func(
 	}
 	return func() (err error) {
 		if err = s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-pre", step)); err != nil {
-			err = fmt.Errorf("withLabeledCommands: %s: error running pre commands: %w", step, err)
+			err = fmt.Errorf("(*script).withLabeledCommands: %s: error running pre commands: %w", step, err)
 			return
 		}
 		defer func() {
-			derr := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step))
+			if derr := s.runLabeledCommands(fmt.Sprintf("docker-volume-backup.%s-post", step)); derr != nil {
-			if err == nil && derr != nil {
+				err = errors.Join(err, fmt.Errorf("(*script).withLabeledCommands: error running %s-post commands: %w", step, derr))
-				err = derr
 			}
 		}()
 		err = cb()

cmd/backup/main.go

@@ -5,230 +5,8 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"log/slog"
-	"os"
-	"os/signal"
-	"runtime"
-	"syscall"
-
-	"github.com/robfig/cron/v3"
 )
 
-type command struct {
-	logger *slog.Logger
-}
-
-func newCommand() *command {
-	return &command{
-		logger: slog.New(slog.NewTextHandler(os.Stdout, nil)),
-	}
-}
-
-func (c *command) must(err error) {
-	if err != nil {
-		c.logger.Error(
-			fmt.Sprintf("Fatal error running command: %v", err),
-			"error",
-			err,
-		)
-		os.Exit(1)
-	}
-}
-
-func runScript(c *Config, envVars map[string]string) (err error) {
-	defer func() {
-		if derr := recover(); derr != nil {
-			err = fmt.Errorf("runScript: unexpected panic running script: %v", err)
-		}
-	}()
-
-	s, unlock, err := newScript(c, envVars)
-	if err != nil {
-		err = fmt.Errorf("runScript: error instantiating script: %w", err)
-		return
-	}
-	defer func() {
-		derr := unlock()
-		if err != nil {
-			err = fmt.Errorf("runScript: error releasing file lock: %w", derr)
-		}
-	}()
-
-	runErr := func() (err error) {
-		scriptErr := func() error {
-			if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) {
-				restartContainersAndServices, err := s.stopContainersAndServices()
-				// The mechanism for restarting containers is not using hooks as it
-				// should happen as soon as possible (i.e. before uploading backups or
-				// similar).
-				defer func() {
-					derr := restartContainersAndServices()
-					if err == nil {
-						err = derr
-					}
-				}()
-				if err != nil {
-					return
-				}
-				err = s.createArchive()
-				return
-			})(); err != nil {
-				return err
-			}
-
-			if err := s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)(); err != nil {
-				return err
-			}
-			if err := s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)(); err != nil {
-				return err
-			}
-			if err := s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)(); err != nil {
-				return err
-			}
-			return nil
-		}()
-
-		if hookErr := s.runHooks(scriptErr); hookErr != nil {
-			if scriptErr != nil {
-				return fmt.Errorf(
-					"runScript: error %w executing the script followed by %w calling the registered hooks",
-					scriptErr,
-					hookErr,
-				)
-			}
-			return fmt.Errorf(
-				"runScript: the script ran successfully, but an error occurred calling the registered hooks: %w",
-				hookErr,
-			)
-		}
-		if scriptErr != nil {
-			return fmt.Errorf("runScript: error running script: %w", scriptErr)
-		}
-		return nil
-	}()
-
-	if runErr != nil {
-		s.logger.Error(
-			fmt.Sprintf("Script run failed: %v", runErr), "error", runErr,
-		)
-	}
-	return runErr
-}
-
-func (c *command) runInForeground(profileCronExpression string) error {
-	cr := cron.New(
-		cron.WithParser(
-			cron.NewParser(
-				cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
-			),
-		),
-	)
-
-	addJob := func(config *Config, name string, envVars map[string]string) error {
-		if _, err := cr.AddFunc(config.BackupCronExpression, func() {
-			c.logger.Info(
-				fmt.Sprintf(
-					"Now running script on schedule %s",
-					config.BackupCronExpression,
-				),
-			)
-
-			if err := runScript(config, envVars); err != nil {
-				c.logger.Error(
-					fmt.Sprintf(
-						"Unexpected error running schedule %s: %v",
-						config.BackupCronExpression,
-						err,
-					),
-					"error",
-					err,
-				)
-			}
-		}); err != nil {
-			return fmt.Errorf("addJob: error adding schedule %s: %w", config.BackupCronExpression, err)
-		}
-
-		c.logger.Info(fmt.Sprintf("Successfully scheduled backup %s with expression %s", name, config.BackupCronExpression))
-		if ok := checkCronSchedule(config.BackupCronExpression); !ok {
-			c.logger.Warn(
-				fmt.Sprintf("Scheduled cron expression %s will never run, is this intentional?", config.BackupCronExpression),
-			)
-		}
-
-		return nil
-	}
-
-	cs, err := loadEnvFiles("/etc/dockervolumebackup/conf.d")
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return fmt.Errorf("runInForeground: could not load config from environment files: %w", err)
-		}
-
-		c, err := loadEnvVars()
-		if err != nil {
-			return fmt.Errorf("runInForeground: could not load config from environment variables: %w", err)
-		} else {
-			err = addJob(c, "from environment", nil)
-			if err != nil {
-				return fmt.Errorf("runInForeground: error adding job from env: %w", err)
-			}
-		}
-	} else {
-		c.logger.Info("/etc/dockervolumebackup/conf.d was found, using configuration files from this directory.")
-		for _, config := range cs {
-			err = addJob(config.config, config.name, config.additionalEnvVars)
-			if err != nil {
-				return fmt.Errorf("runInForeground: error adding jobs from conf files: %w", err)
-			}
-		}
-	}
-
-	if profileCronExpression != "" {
-		if _, err := cr.AddFunc(profileCronExpression, func() {
-			memStats := runtime.MemStats{}
-			runtime.ReadMemStats(&memStats)
-			c.logger.Info(
-				"Collecting runtime information",
-				"num_goroutines",
-				runtime.NumGoroutine(),
-				"memory_heap_alloc",
-				formatBytes(memStats.HeapAlloc, false),
-				"memory_heap_inuse",
-				formatBytes(memStats.HeapInuse, false),
-				"memory_heap_sys",
-				formatBytes(memStats.HeapSys, false),
-				"memory_heap_objects",
-				memStats.HeapObjects,
-			)
-		}); err != nil {
-			return fmt.Errorf("runInForeground: error adding profiling job: %w", err)
-		}
-	}
-
-	var quit = make(chan os.Signal, 1)
-	signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT)
-	cr.Start()
-	<-quit
-	ctx := cr.Stop()
-	<-ctx.Done()
-
-	return nil
-}
-
-func (c *command) runAsCommand() error {
-	config, err := loadEnvVars()
-	if err != nil {
-		return fmt.Errorf("runAsCommand: error loading env vars: %w", err)
-	}
-	err = runScript(config, nil)
-	if err != nil {
-		return fmt.Errorf("runAsCommand: error running script: %w", err)
-	}
-
-	return nil
-}
-
 func main() {
 	foreground := flag.Bool("foreground", false, "run the tool in the foreground")
 	profile := flag.String("profile", "", "collect runtime metrics and log them periodically on the given cron expression")
@@ -236,7 +14,10 @@ func main() {
 
 	c := newCommand()
 	if *foreground {
-		c.must(c.runInForeground(*profile))
+		opts := foregroundOpts{
+			profileCronExpression: *profile,
+		}
+		c.must(c.runInForeground(opts))
 	} else {
 		c.must(c.runAsCommand())
 	}

cmd/backup/profile.go

@@ -0,0 +1,24 @@
+// Copyright 2024 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import "runtime"
+
+func (c *command) profile() {
+	memStats := runtime.MemStats{}
+	runtime.ReadMemStats(&memStats)
+	c.logger.Info(
+		"Collecting runtime information",
+		"num_goroutines",
+		runtime.NumGoroutine(),
+		"memory_heap_alloc",
+		formatBytes(memStats.HeapAlloc, false),
+		"memory_heap_inuse",
+		formatBytes(memStats.HeapInuse, false),
+		"memory_heap_sys",
+		formatBytes(memStats.HeapSys, false),
+		"memory_heap_objects",
+		memStats.HeapObjects,
+	)
+}

cmd/backup/prune_backups.go

@@ -0,0 +1,65 @@
+// Copyright 2024 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// pruneBackups rotates away backups from local and remote storages using
+// the given configuration. In case the given configuration would delete all
+// backups, it does nothing instead and logs a warning.
+func (s *script) pruneBackups() error {
+	if s.c.BackupRetentionDays < 0 {
+		return nil
+	}
+
+	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
+
+	eg := errgroup.Group{}
+	for _, backend := range s.storages {
+		b := backend
+		eg.Go(func() error {
+			if skipPrune(b.Name(), s.c.BackupSkipBackendsFromPrune) {
+				s.logger.Info(
+					fmt.Sprintf("Skipping pruning for backend `%s`.", b.Name()),
+				)
+				return nil
+			}
+			stats, err := b.Prune(deadline, s.c.BackupPruningPrefix)
+			if err != nil {
+				return err
+			}
+			s.stats.Lock()
+			s.stats.Storages[b.Name()] = StorageStats{
+				Total:  stats.Total,
+				Pruned: stats.Pruned,
+			}
+			s.stats.Unlock()
+			return nil
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		return fmt.Errorf("pruneBackups: error pruning backups: %w", err)
+	}
+
+	return nil
+}
+
+// skipPrune returns true if the given backend name is contained in the
+// list of skipped backends.
+func skipPrune(name string, skippedBackends []string) bool {
+	return slices.ContainsFunc(
+		skippedBackends,
+		func(b string) bool {
+			return strings.EqualFold(b, name) // ignore case on both sides
+		},
+	)
+}

cmd/backup/run_script.go

@@ -0,0 +1,101 @@
+// Copyright 2024 - Offen Authors <hioffen@posteo.de>
+// SPDX-License-Identifier: MPL-2.0
+
+package main
+
+import (
+	"errors"
+	"fmt"
+)
+
+// runScript instantiates a new script object and orchestrates a backup run.
+// To ensure it runs mutually exclusive a global file lock is acquired before
+// it starts running. Any panic within the script will be recovered and returned
+// as an error.
+func runScript(c *Config) (err error) {
+	defer func() {
+		if derr := recover(); derr != nil {
+			err = fmt.Errorf("runScript: unexpected panic running script: %v", derr)
+		}
+	}()
+
+	s := newScript(c)
+
+	unlock, lockErr := s.lock("/var/lock/dockervolumebackup.lock")
+	if lockErr != nil {
+		err = fmt.Errorf("runScript: error acquiring file lock: %w", lockErr)
+		return
+	}
+	defer func() {
+		if derr := unlock(); derr != nil {
+			err = errors.Join(err, fmt.Errorf("runScript: error releasing file lock: %w", derr))
+		}
+	}()
+
+	unset, err := s.c.applyEnv()
+	if err != nil {
+		return fmt.Errorf("runScript: error applying env: %w", err)
+	}
+	defer func() {
+		if derr := unset(); derr != nil {
+			err = errors.Join(err, fmt.Errorf("runScript: error unsetting environment variables: %w", derr))
+		}
+	}()
+
+	if initErr := s.init(); initErr != nil {
+		err = fmt.Errorf("runScript: error instantiating script: %w", initErr)
+		return
+	}
+
+	return func() (err error) {
+		scriptErr := func() error {
+			if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) {
+				restartContainersAndServices, err := s.stopContainersAndServices()
+				// The mechanism for restarting containers is not using hooks as it
+				// should happen as soon as possible (i.e. before uploading backups or
+				// similar).
+				defer func() {
+					if derr := restartContainersAndServices(); derr != nil {
+						err = errors.Join(err, fmt.Errorf("runScript: error restarting containers and services: %w", derr))
+					}
+				}()
+				if err != nil {
+					return
+				}
+				err = s.createArchive()
+				return
+			})(); err != nil {
+				return err
+			}
+
+			if err := s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)(); err != nil {
+				return err
+			}
+			if err := s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)(); err != nil {
+				return err
+			}
+			if err := s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)(); err != nil {
+				return err
+			}
+			return nil
+		}()
+
+		if hookErr := s.runHooks(scriptErr); hookErr != nil {
+			if scriptErr != nil {
+				return fmt.Errorf(
+					"runScript: error %w executing the script followed by %w calling the registered hooks",
+					scriptErr,
+					hookErr,
+				)
+			}
+			return fmt.Errorf(
+				"runScript: the script ran successfully, but an error occurred calling the registered hooks: %w",
+				hookErr,
+			)
+		}
+		if scriptErr != nil {
+			return fmt.Errorf("runScript: error running script: %w", scriptErr)
+		}
+		return nil
+	}()
+}

cmd/backup/script.go

@@ -6,14 +6,9 @@ package main
 import (
 	"bytes"
 	"fmt"
-	"io"
-	"io/fs"
 	"log/slog"
 	"os"
 	"path"
-	"path/filepath"
-	"slices"
-	"strings"
 	"text/template"
 	"time"
 
@@ -25,13 +20,10 @@ import (
 	"github.com/offen/docker-volume-backup/internal/storage/ssh"
 	"github.com/offen/docker-volume-backup/internal/storage/webdav"
 
-	openpgp "github.com/ProtonMail/go-crypto/openpgp/v2"
 	"github.com/containrrr/shoutrrr"
 	"github.com/containrrr/shoutrrr/pkg/router"
 	"github.com/docker/docker/client"
 	"github.com/leekchan/timeutil"
-	"github.com/otiai10/copy"
-	"golang.org/x/sync/errgroup"
 )
 
 // script holds all the stateful information required to orchestrate a
@@ -57,9 +49,9 @@ type script struct {
 // remote resources like the Docker engine or remote storage locations. All
 // reading from env vars or other configuration sources is expected to happen
 // in this method.
-func newScript(c *Config, envVars map[string]string) (*script, func() error, error) {
+func newScript(c *Config) *script {
 	stdOut, logBuffer := buffer(os.Stdout)
-	s := &script{
+	return &script{
 		c:      c,
 		logger: slog.New(slog.NewTextHandler(stdOut, nil)),
 		stats: &Stats{
@@ -75,30 +67,9 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 			},
 		},
 	}
+}
 
-	unlock, err := s.lock("/var/lock/dockervolumebackup.lock")
+func (s *script) init() error {
-	if err != nil {
-		return nil, noop, fmt.Errorf("runScript: error acquiring file lock: %w", err)
-	}
-
-	for key, value := range envVars {
-		currentVal, currentOk := os.LookupEnv(key)
-		defer func(currentKey, currentVal string, currentOk bool) {
-			if !currentOk {
-				_ = os.Unsetenv(currentKey)
-			} else {
-				_ = os.Setenv(currentKey, currentVal)
-			}
-		}(key, currentVal, currentOk)
-
-		if err := os.Setenv(key, value); err != nil {
-			return nil, unlock, fmt.Errorf(
-				"Unexpected error overloading environment %s: %w",
-				s.c.BackupCronExpression,
-				err,
-			)
-		}
-	}
 	s.registerHook(hookLevelPlumbing, func(error) error {
 		s.stats.EndTime = time.Now()
 		s.stats.TookTime = s.stats.EndTime.Sub(s.stats.StartTime)
@@ -109,14 +80,14 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 
 	tmplFileName, tErr := template.New("extension").Parse(s.file)
 	if tErr != nil {
-		return nil, unlock, fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr)
+		return fmt.Errorf("newScript: unable to parse backup file extension template: %w", tErr)
 	}
 
 	var bf bytes.Buffer
 	if tErr := tmplFileName.Execute(&bf, map[string]string{
 		"Extension": fmt.Sprintf("tar.%s", s.c.BackupCompression),
 	}); tErr != nil {
-		return nil, unlock, fmt.Errorf("newScript: error executing backup file extension template: %w", tErr)
+		return fmt.Errorf("newScript: error executing backup file extension template: %w", tErr)
 	}
 	s.file = bf.String()
 
@@ -127,12 +98,12 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 	}
 	s.file = timeutil.Strftime(&s.stats.StartTime, s.file)
 
-	_, err = os.Stat("/var/run/docker.sock")
+	_, err := os.Stat("/var/run/docker.sock")
 	_, dockerHostSet := os.LookupEnv("DOCKER_HOST")
 	if !os.IsNotExist(err) || dockerHostSet {
 		cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
 		if err != nil {
-			return nil, unlock, fmt.Errorf("newScript: failed to create docker client")
+			return fmt.Errorf("newScript: failed to create docker client")
 		}
 		s.cli = cli
 		s.registerHook(hookLevelPlumbing, func(err error) error {
@@ -170,7 +141,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 		}
 		s3Backend, err := s3.NewStorageBackend(s3Config, logFunc)
 		if err != nil {
-			return nil, unlock, fmt.Errorf("newScript: error creating s3 storage backend: %w", err)
+			return fmt.Errorf("newScript: error creating s3 storage backend: %w", err)
 		}
 		s.storages = append(s.storages, s3Backend)
 	}
@@ -185,7 +156,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 		}
 		webdavBackend, err := webdav.NewStorageBackend(webDavConfig, logFunc)
 		if err != nil {
-			return nil, unlock, fmt.Errorf("newScript: error creating webdav storage backend: %w", err)
+			return fmt.Errorf("newScript: error creating webdav storage backend: %w", err)
 		}
 		s.storages = append(s.storages, webdavBackend)
 	}
@@ -202,7 +173,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 		}
 		sshBackend, err := ssh.NewStorageBackend(sshConfig, logFunc)
 		if err != nil {
-			return nil, unlock, fmt.Errorf("newScript: error creating ssh storage backend: %w", err)
+			return fmt.Errorf("newScript: error creating ssh storage backend: %w", err)
 		}
 		s.storages = append(s.storages, sshBackend)
 	}
@@ -226,7 +197,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 		}
 		azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
 		if err != nil {
-			return nil, unlock, fmt.Errorf("newScript: error creating azure storage backend: %w", err)
+			return fmt.Errorf("newScript: error creating azure storage backend: %w", err)
 		}
 		s.storages = append(s.storages, azureBackend)
 	}
@@ -243,7 +214,7 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 		}
 		dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc)
 		if err != nil {
-			return nil, unlock, fmt.Errorf("newScript: error creating dropbox storage backend: %w", err)
+			return fmt.Errorf("newScript: error creating dropbox storage backend: %w", err)
 		}
 		s.storages = append(s.storages, dropboxBackend)
 	}
@@ -269,14 +240,14 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 
 	hookLevel, ok := hookLevels[s.c.NotificationLevel]
 	if !ok {
-		return nil, unlock, fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel)
+		return fmt.Errorf("newScript: unknown NOTIFICATION_LEVEL %s", s.c.NotificationLevel)
 	}
 	s.hookLevel = hookLevel
 
 	if len(s.c.NotificationURLs) > 0 {
 		sender, senderErr := shoutrrr.CreateSender(s.c.NotificationURLs...)
 		if senderErr != nil {
-			return nil, unlock, fmt.Errorf("newScript: error creating sender: %w", senderErr)
+			return fmt.Errorf("newScript: error creating sender: %w", senderErr)
 		}
 		s.sender = sender
 
@@ -284,13 +255,13 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 		tmpl.Funcs(templateHelpers)
 		tmpl, err = tmpl.Parse(defaultNotifications)
 		if err != nil {
-			return nil, unlock, fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
+			return fmt.Errorf("newScript: unable to parse default notifications templates: %w", err)
 		}
 
 		if fi, err := os.Stat("/etc/dockervolumebackup/notifications.d"); err == nil && fi.IsDir() {
 			tmpl, err = tmpl.ParseGlob("/etc/dockervolumebackup/notifications.d/*.*")
 			if err != nil {
-				return nil, unlock, fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
+				return fmt.Errorf("newScript: unable to parse user defined notifications templates: %w", err)
 			}
 		}
 		s.template = tmpl
@@ -311,211 +282,5 @@ func newScript(c *Config, envVars map[string]string) (*script, func() error, err
 		})
 	}
 
-	return s, unlock, nil
-}
-
-// createArchive creates a tar archive of the configured backup location and
-// saves it to disk.
-func (s *script) createArchive() error {
-	backupSources := s.c.BackupSources
-
-	if s.c.BackupFromSnapshot {
-		s.logger.Warn(
-			"Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.",
-		)
-		s.logger.Warn(
-			"Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the documentation for an upgrade guide.",
-		)
-		backupSources = filepath.Join("/tmp", s.c.BackupSources)
-		// copy before compressing guard against a situation where backup folder's content are still growing.
-		s.registerHook(hookLevelPlumbing, func(error) error {
-			if err := remove(backupSources); err != nil {
-				return fmt.Errorf("createArchive: error removing snapshot: %w", err)
-			}
-			s.logger.Info(
-				fmt.Sprintf("Removed snapshot `%s`.", backupSources),
-			)
-			return nil
-		})
-		if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{
-			PreserveTimes: true,
-			PreserveOwner: true,
-		}); err != nil {
-			return fmt.Errorf("createArchive: error creating snapshot: %w", err)
-		}
-		s.logger.Info(
-			fmt.Sprintf("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources),
-		)
-	}
-
-	tarFile := s.file
-	s.registerHook(hookLevelPlumbing, func(error) error {
-		if err := remove(tarFile); err != nil {
-			return fmt.Errorf("createArchive: error removing tar file: %w", err)
-		}
-		s.logger.Info(
-			fmt.Sprintf("Removed tar file `%s`.", tarFile),
-		)
-		return nil
-	})
-
-	backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources))
-	if err != nil {
-		return fmt.Errorf("createArchive: error getting absolute path: %w", err)
-	}
-
-	var filesEligibleForBackup []string
-	if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) {
-			return nil
-		}
-		filesEligibleForBackup = append(filesEligibleForBackup, path)
-		return nil
-	}); err != nil {
-		return fmt.Errorf("createArchive: error walking filesystem tree: %w", err)
-	}
-
-	if err := createArchive(filesEligibleForBackup, backupSources, tarFile, s.c.BackupCompression.String(), s.c.GzipParallelism.Int()); err != nil {
-		return fmt.Errorf("createArchive: error compressing backup folder: %w", err)
-	}
-
-	s.logger.Info(
-		fmt.Sprintf("Created backup of `%s` at `%s`.", backupSources, tarFile),
-	)
 	return nil
 }
-
-// encryptArchive encrypts the backup file using PGP and the configured passphrase.
-// In case no passphrase is given it returns early, leaving the backup file
-// untouched.
-func (s *script) encryptArchive() error {
-	if s.c.GpgPassphrase == "" {
-		return nil
-	}
-
-	gpgFile := fmt.Sprintf("%s.gpg", s.file)
-	s.registerHook(hookLevelPlumbing, func(error) error {
-		if err := remove(gpgFile); err != nil {
-			return fmt.Errorf("encryptArchive: error removing gpg file: %w", err)
-		}
-		s.logger.Info(
-			fmt.Sprintf("Removed GPG file `%s`.", gpgFile),
-		)
-		return nil
-	})
-
-	outFile, err := os.Create(gpgFile)
-	if err != nil {
-		return fmt.Errorf("encryptArchive: error opening out file: %w", err)
-	}
-	defer outFile.Close()
-
-	_, name := path.Split(s.file)
-	dst, err := openpgp.SymmetricallyEncrypt(outFile, []byte(s.c.GpgPassphrase), &openpgp.FileHints{
-		FileName: name,
-	}, nil)
-	if err != nil {
-		return fmt.Errorf("encryptArchive: error encrypting backup file: %w", err)
-	}
-	defer dst.Close()
-
-	src, err := os.Open(s.file)
-	if err != nil {
-		return fmt.Errorf("encryptArchive: error opening backup file `%s`: %w", s.file, err)
-	}
-
-	if _, err := io.Copy(dst, src); err != nil {
-		return fmt.Errorf("encryptArchive: error writing ciphertext to file: %w", err)
-	}
-
-	s.file = gpgFile
-	s.logger.Info(
-		fmt.Sprintf("Encrypted backup using given passphrase, saving as `%s`.", s.file),
-	)
-	return nil
-}
-
-// copyArchive makes sure the backup file is copied to both local and remote locations
-// as per the given configuration.
-func (s *script) copyArchive() error {
-	_, name := path.Split(s.file)
-	if stat, err := os.Stat(s.file); err != nil {
-		return fmt.Errorf("copyArchive: unable to stat backup file: %w", err)
-	} else {
-		size := stat.Size()
-		s.stats.BackupFile = BackupFileStats{
-			Size:     uint64(size),
-			Name:     name,
-			FullPath: s.file,
-		}
-	}
-
-	eg := errgroup.Group{}
-	for _, backend := range s.storages {
-		b := backend
-		eg.Go(func() error {
-			return b.Copy(s.file)
-		})
-	}
-	if err := eg.Wait(); err != nil {
-		return fmt.Errorf("copyArchive: error copying archive: %w", err)
-	}
-
-	return nil
-}
-
-// pruneBackups rotates away backups from local and remote storages using
-// the given configuration. In case the given configuration would delete all
-// backups, it does nothing instead and logs a warning.
-func (s *script) pruneBackups() error {
-	if s.c.BackupRetentionDays < 0 {
-		return nil
-	}
-
-	deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
-
-	eg := errgroup.Group{}
-	for _, backend := range s.storages {
-		b := backend
-		eg.Go(func() error {
-			if skipPrune(b.Name(), s.c.BackupSkipBackendsFromPrune) {
-				s.logger.Info(
-					fmt.Sprintf("Skipping pruning for backend `%s`.", b.Name()),
-				)
-				return nil
-			}
-			stats, err := b.Prune(deadline, s.c.BackupPruningPrefix)
-			if err != nil {
-				return err
-			}
-			s.stats.Lock()
-			s.stats.Storages[b.Name()] = StorageStats{
-				Total:  stats.Total,
-				Pruned: stats.Pruned,
-			}
-			s.stats.Unlock()
-			return nil
-		})
-	}
-
-	if err := eg.Wait(); err != nil {
-		return fmt.Errorf("pruneBackups: error pruning backups: %w", err)
-	}
-
-	return nil
-}
-
-// skipPrune returns true if the given backend name is contained in the
-// list of skipped backends.
-func skipPrune(name string, skippedBackends []string) bool {
-	return slices.ContainsFunc(
-		skippedBackends,
-		func(b string) bool {
-			return strings.EqualFold(b, name) // ignore case on both sides
-		},
-	)
-}

cmd/backup/util.go

@@ -9,6 +9,9 @@ import (
 	"io"
 	"os"
 	"sync"
+	"time"
+
+	"github.com/robfig/cron/v3"
 )
 
 var noop = func() error { return nil }
@@ -79,3 +82,22 @@ func (c *concurrentSlice[T]) append(v T) {
 func (c *concurrentSlice[T]) value() []T {
 	return c.val
 }
+
+// checkCronSchedule detects whether the given cron expression will actually
+// ever be executed or not.
+func checkCronSchedule(expression string) (ok bool) {
+	defer func() {
+		if err := recover(); err != nil {
+			ok = false
+		}
+	}()
+	sched, err := cron.ParseStandard(expression)
+	if err != nil {
+		ok = false
+		return
+	}
+	now := time.Now()
+	sched.Next(now) // panics when the cron would never run
+	ok = true
+	return
+}