From 411c39ee722d87a77ac63fe54607b63dd3f822e4 Mon Sep 17 00:00:00 2001 From: schwannden Date: Mon, 8 Nov 2021 15:39:18 +0800 Subject: [PATCH] create a snapshot before creating tar archive (#32) * create a snapshot before creating tar archive * safeguard snapshot removal and make snapshot optional * fix typo, make sure remove snapshot failure triggers failure hook Co-authored-by: Schwannden Kuo --- README.md | 7 +++++++ cmd/backup/main.go | 32 +++++++++++++++++++++++++++----- go.mod | 4 +++- go.sum | 9 +++++++++ 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index cc2f209..1c9c019 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,13 @@ You can populate below template according to your requirements and use it as you # BACKUP_LATEST_SYMLINK="backup.latest.tar.gz" +# Wehter to copy the content of backup folder before creating archive. +# In rare scenario where the content of the source backup volume is still +# updating, but we do not wish to stop the container while performing backup, +# this snapshot before archive ensures the integrity of the tar.gz file. + +# BACKUP_FROM_SNAPSHOT="false" + ########### BACKUP STORAGE # The name of the remote bucket that should be used for storing backups. If diff --git a/cmd/backup/main.go b/cmd/backup/main.go index a920d2c..a63ba9e 100644 --- a/cmd/backup/main.go +++ b/cmd/backup/main.go @@ -26,6 +26,7 @@ import ( "github.com/m90/targz" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/otiai10/copy" "github.com/sirupsen/logrus" "golang.org/x/crypto/openpgp" ) @@ -95,6 +96,7 @@ type config struct { BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` BackupPruningPrefix string `split_words:"true"` BackupStopContainerLabel string `split_words:"true" default:"true"` + BackupFromSnapshot bool `split_words:"true"` AwsS3BucketName string `split_words:"true"` AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` AwsEndpointProto string `split_words:"true" default:"https"` @@ -326,14 +328,27 @@ func (s *script) stopContainers() (func() error, error) { }, stopError } +func (s *script) snapshotFolder() string { + return filepath.Join("/tmp", s.c.BackupSources) +} + // takeBackup creates a tar archive of the configured backup location and // saves it to disk. func (s *script) takeBackup() error { s.file = timeutil.Strftime(&s.start, s.file) - if err := targz.Compress(s.c.BackupSources, s.file); err != nil { + backupSources := s.c.BackupSources + if s.c.BackupFromSnapshot { + backupSources = s.snapshotFolder() + s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources) + // copy before compressing guard against a situation where backup folder's content are still growing. + if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{ PreserveTimes: true }); err != nil { + return fmt.Errorf("takeBackup: error creating snapshot on backup folder: %w", err) + } + } + if err := targz.Compress(backupSources, s.file); err != nil { return fmt.Errorf("takeBackup: error compressing backup folder: %w", err) } - s.logger.Infof("Created backup of `%s` at `%s`.", s.c.BackupSources, s.file) + s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, s.file) return nil } @@ -392,7 +407,7 @@ func (s *script) copyBackup() error { } if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) { - if err := copy(s.file, path.Join(s.c.BackupArchive, name)); err != nil { + if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil { return fmt.Errorf("copyBackup: error copying file to local archive: %w", err) } s.logger.Infof("Stored copy of backup `%s` in local archive `%s`.", s.file, s.c.BackupArchive) @@ -410,8 +425,15 @@ func (s *script) copyBackup() error { return nil } -// removeArtifacts removes the backup file from disk. +// removeArtifacts removes the backup file from disk. Also remove snapshot if snapshot was created func (s *script) removeArtifacts() error { + if s.c.BackupFromSnapshot { + snapshotFolder := s.snapshotFolder() + if err := os.RemoveAll(snapshotFolder); err != nil { + return fmt.Errorf("removeArtifacts: error removing snapshot folder %s: %w", snapshotFolder, err) + } + s.logger.Infof("Removed snapshot folder %s.", snapshotFolder) + } _, err := os.Stat(s.file) if err != nil { if os.IsNotExist(err) { @@ -628,7 +650,7 @@ func lock(lockfile string) func() error { } // copy creates a copy of the file located at `dst` at `src`. -func copy(src, dst string) error { +func copyFile(src, dst string) error { in, err := os.Open(src) if err != nil { return err diff --git a/go.mod b/go.mod index b07a5a6..1760155 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,13 @@ go 1.17 require ( github.com/docker/docker v20.10.8+incompatible + github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df github.com/gofrs/flock v0.8.1 github.com/kelseyhightower/envconfig v1.4.0 github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615 github.com/minio/minio-go/v7 v7.0.12 + github.com/otiai10/copy v1.6.0 github.com/sirupsen/logrus v1.8.1 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 ) @@ -20,7 +22,6 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.0 // indirect github.com/google/uuid v1.2.0 // indirect @@ -43,5 +44,6 @@ require ( google.golang.org/grpc v1.33.2 // indirect google.golang.org/protobuf v1.26.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect gopkg.in/ini.v1 v1.57.0 // indirect ) diff --git a/go.sum b/go.sum index babacaf..40fbf22 100644 --- a/go.sum +++ b/go.sum @@ -494,6 +494,13 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ= +github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E= +github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -921,6 +928,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=