mirror of
https://github.com/offen/docker-volume-backup.git
synced 2024-11-10 00:30:29 +01:00
create a snapshot before creating tar archive (#32)
* create a snapshot before creating tar archive * safeguard snapshot removal and make snapshot optional * fix typo, make sure remove snapshot failure triggers failure hook Co-authored-by: Schwannden Kuo <schwannden@mobagel.com>
This commit is contained in:
parent
0c666d0c88
commit
411c39ee72
@ -126,6 +126,13 @@ You can populate below template according to your requirements and use it as you
|
|||||||
|
|
||||||
# BACKUP_LATEST_SYMLINK="backup.latest.tar.gz"
|
# BACKUP_LATEST_SYMLINK="backup.latest.tar.gz"
|
||||||
|
|
||||||
|
# Wehter to copy the content of backup folder before creating archive.
|
||||||
|
# In rare scenario where the content of the source backup volume is still
|
||||||
|
# updating, but we do not wish to stop the container while performing backup,
|
||||||
|
# this snapshot before archive ensures the integrity of the tar.gz file.
|
||||||
|
|
||||||
|
# BACKUP_FROM_SNAPSHOT="false"
|
||||||
|
|
||||||
########### BACKUP STORAGE
|
########### BACKUP STORAGE
|
||||||
|
|
||||||
# The name of the remote bucket that should be used for storing backups. If
|
# The name of the remote bucket that should be used for storing backups. If
|
||||||
|
@ -26,6 +26,7 @@ import (
|
|||||||
"github.com/m90/targz"
|
"github.com/m90/targz"
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||||
|
"github.com/otiai10/copy"
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
"golang.org/x/crypto/openpgp"
|
"golang.org/x/crypto/openpgp"
|
||||||
)
|
)
|
||||||
@ -95,6 +96,7 @@ type config struct {
|
|||||||
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
|
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
|
||||||
BackupPruningPrefix string `split_words:"true"`
|
BackupPruningPrefix string `split_words:"true"`
|
||||||
BackupStopContainerLabel string `split_words:"true" default:"true"`
|
BackupStopContainerLabel string `split_words:"true" default:"true"`
|
||||||
|
BackupFromSnapshot bool `split_words:"true"`
|
||||||
AwsS3BucketName string `split_words:"true"`
|
AwsS3BucketName string `split_words:"true"`
|
||||||
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
|
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
|
||||||
AwsEndpointProto string `split_words:"true" default:"https"`
|
AwsEndpointProto string `split_words:"true" default:"https"`
|
||||||
@ -326,14 +328,27 @@ func (s *script) stopContainers() (func() error, error) {
|
|||||||
}, stopError
|
}, stopError
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *script) snapshotFolder() string {
|
||||||
|
return filepath.Join("/tmp", s.c.BackupSources)
|
||||||
|
}
|
||||||
|
|
||||||
// takeBackup creates a tar archive of the configured backup location and
|
// takeBackup creates a tar archive of the configured backup location and
|
||||||
// saves it to disk.
|
// saves it to disk.
|
||||||
func (s *script) takeBackup() error {
|
func (s *script) takeBackup() error {
|
||||||
s.file = timeutil.Strftime(&s.start, s.file)
|
s.file = timeutil.Strftime(&s.start, s.file)
|
||||||
if err := targz.Compress(s.c.BackupSources, s.file); err != nil {
|
backupSources := s.c.BackupSources
|
||||||
|
if s.c.BackupFromSnapshot {
|
||||||
|
backupSources = s.snapshotFolder()
|
||||||
|
s.logger.Infof("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources)
|
||||||
|
// copy before compressing guard against a situation where backup folder's content are still growing.
|
||||||
|
if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{ PreserveTimes: true }); err != nil {
|
||||||
|
return fmt.Errorf("takeBackup: error creating snapshot on backup folder: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := targz.Compress(backupSources, s.file); err != nil {
|
||||||
return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
|
return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
|
||||||
}
|
}
|
||||||
s.logger.Infof("Created backup of `%s` at `%s`.", s.c.BackupSources, s.file)
|
s.logger.Infof("Created backup of `%s` at `%s`.", backupSources, s.file)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -392,7 +407,7 @@ func (s *script) copyBackup() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
|
if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
|
||||||
if err := copy(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
|
if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
|
||||||
return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
|
return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
|
||||||
}
|
}
|
||||||
s.logger.Infof("Stored copy of backup `%s` in local archive `%s`.", s.file, s.c.BackupArchive)
|
s.logger.Infof("Stored copy of backup `%s` in local archive `%s`.", s.file, s.c.BackupArchive)
|
||||||
@ -410,8 +425,15 @@ func (s *script) copyBackup() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeArtifacts removes the backup file from disk.
|
// removeArtifacts removes the backup file from disk. Also remove snapshot if snapshot was created
|
||||||
func (s *script) removeArtifacts() error {
|
func (s *script) removeArtifacts() error {
|
||||||
|
if s.c.BackupFromSnapshot {
|
||||||
|
snapshotFolder := s.snapshotFolder()
|
||||||
|
if err := os.RemoveAll(snapshotFolder); err != nil {
|
||||||
|
return fmt.Errorf("removeArtifacts: error removing snapshot folder %s: %w", snapshotFolder, err)
|
||||||
|
}
|
||||||
|
s.logger.Infof("Removed snapshot folder %s.", snapshotFolder)
|
||||||
|
}
|
||||||
_, err := os.Stat(s.file)
|
_, err := os.Stat(s.file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
@ -628,7 +650,7 @@ func lock(lockfile string) func() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// copy creates a copy of the file located at `dst` at `src`.
|
// copy creates a copy of the file located at `dst` at `src`.
|
||||||
func copy(src, dst string) error {
|
func copyFile(src, dst string) error {
|
||||||
in, err := os.Open(src)
|
in, err := os.Open(src)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
4
go.mod
4
go.mod
@ -4,11 +4,13 @@ go 1.17
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/docker/docker v20.10.8+incompatible
|
github.com/docker/docker v20.10.8+incompatible
|
||||||
|
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df
|
||||||
github.com/gofrs/flock v0.8.1
|
github.com/gofrs/flock v0.8.1
|
||||||
github.com/kelseyhightower/envconfig v1.4.0
|
github.com/kelseyhightower/envconfig v1.4.0
|
||||||
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
|
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
|
||||||
github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615
|
github.com/m90/targz v0.0.0-20210904082215-2e9a4529a615
|
||||||
github.com/minio/minio-go/v7 v7.0.12
|
github.com/minio/minio-go/v7 v7.0.12
|
||||||
|
github.com/otiai10/copy v1.6.0
|
||||||
github.com/sirupsen/logrus v1.8.1
|
github.com/sirupsen/logrus v1.8.1
|
||||||
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
|
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
|
||||||
)
|
)
|
||||||
@ -20,7 +22,6 @@ require (
|
|||||||
github.com/docker/go-connections v0.4.0 // indirect
|
github.com/docker/go-connections v0.4.0 // indirect
|
||||||
github.com/docker/go-units v0.4.0 // indirect
|
github.com/docker/go-units v0.4.0 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.0 // indirect
|
github.com/dustin/go-humanize v1.0.0 // indirect
|
||||||
github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df // indirect
|
|
||||||
github.com/gogo/protobuf v1.3.2 // indirect
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
github.com/golang/protobuf v1.5.0 // indirect
|
github.com/golang/protobuf v1.5.0 // indirect
|
||||||
github.com/google/uuid v1.2.0 // indirect
|
github.com/google/uuid v1.2.0 // indirect
|
||||||
@ -43,5 +44,6 @@ require (
|
|||||||
google.golang.org/grpc v1.33.2 // indirect
|
google.golang.org/grpc v1.33.2 // indirect
|
||||||
google.golang.org/protobuf v1.26.0 // indirect
|
google.golang.org/protobuf v1.26.0 // indirect
|
||||||
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
|
||||||
|
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df // indirect
|
||||||
gopkg.in/ini.v1 v1.57.0 // indirect
|
gopkg.in/ini.v1 v1.57.0 // indirect
|
||||||
)
|
)
|
||||||
|
9
go.sum
9
go.sum
@ -494,6 +494,13 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo
|
|||||||
github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
|
github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
|
||||||
github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
|
github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
|
||||||
github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
|
github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
|
||||||
|
github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ=
|
||||||
|
github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E=
|
||||||
|
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
|
||||||
|
github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
|
||||||
|
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
|
||||||
|
github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E=
|
||||||
|
github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
|
||||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||||
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
|
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
|
||||||
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
|
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
|
||||||
@ -921,6 +928,8 @@ gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS
|
|||||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||||
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
|
||||||
|
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE=
|
||||||
|
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw=
|
||||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||||
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
|
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
|
||||||
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||||
|
Loading…
Reference in New Issue
Block a user