From 4e04cb9926fc7b1415d4a2acb4f9571f7bec4569 Mon Sep 17 00:00:00 2001 From: MaxJa4 <74194322+MaxJa4@users.noreply.github.com> Date: Sun, 20 Aug 2023 16:29:35 +0200 Subject: [PATCH] Add new storage backend: Dropbox (#103) --- cmd/backup/config.go | 2 + cmd/backup/script.go | 24 +++- go.mod | 8 ++ go.sum | 4 + internal/storage/dropbox/dropbox.go | 182 ++++++++++++++++++++++++++++ 5 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 internal/storage/dropbox/dropbox.go diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 7f62afe..b297304 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -70,6 +70,8 @@ type Config struct { AzureStorageContainerName string `split_words:"true"` AzureStoragePath string `split_words:"true"` AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` + DropboxToken string `split_words:"true"` + DropboxRemotePath string `split_words:"true"` } func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { diff --git a/cmd/backup/script.go b/cmd/backup/script.go index ca2ce9c..c074105 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -19,6 +19,7 @@ import ( "github.com/offen/docker-volume-backup/internal/storage" "github.com/offen/docker-volume-backup/internal/storage/azure" + "github.com/offen/docker-volume-backup/internal/storage/dropbox" "github.com/offen/docker-volume-backup/internal/storage/local" "github.com/offen/docker-volume-backup/internal/storage/s3" "github.com/offen/docker-volume-backup/internal/storage/ssh" @@ -70,11 +71,12 @@ func newScript() (*script, error) { StartTime: time.Now(), LogOutput: logBuffer, Storages: map[string]StorageStats{ - "S3": {}, - "WebDAV": {}, - "SSH": {}, - "Local": {}, - "Azure": {}, + "S3": {}, + "WebDAV": {}, + "SSH": {}, + "Local": {}, + "Azure": {}, + "Dropbox": {}, }, }, } @@ -218,6 +220,18 @@ func newScript() (*script, error) { s.storages = append(s.storages, azureBackend) } + if s.c.DropboxToken != "" { + dropboxConfig := dropbox.Config{ + Token: s.c.DropboxToken, + RemotePath: s.c.DropboxRemotePath, + } + dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc) + if err != nil { + return nil, err + } + s.storages = append(s.storages, dropboxBackend) + } + if s.c.EmailNotificationRecipient != "" { emailURL := fmt.Sprintf( "smtp://%s:%s@%s:%d/?from=%s&to=%s", diff --git a/go.mod b/go.mod index 3b75292..61e11b9 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,13 @@ require ( golang.org/x/sync v0.3.0 ) +require ( + github.com/golang/protobuf v1.5.2 // indirect + golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.28.1 // indirect +) + require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect @@ -28,6 +35,7 @@ require ( github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect + github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.13.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect diff --git a/go.sum b/go.sum index a9aba53..44e1ac1 100644 --- a/go.sum +++ b/go.sum @@ -257,6 +257,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU= +github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -785,6 +787,7 @@ golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7Lm golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1046,6 +1049,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/internal/storage/dropbox/dropbox.go b/internal/storage/dropbox/dropbox.go new file mode 100644 index 0000000..d5302bb --- /dev/null +++ b/internal/storage/dropbox/dropbox.go @@ -0,0 +1,182 @@ +package dropbox + +import ( + "bytes" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox" + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files" + "github.com/offen/docker-volume-backup/internal/storage" +) + +type dropboxStorage struct { + *storage.StorageBackend + client files.Client +} + +// Config allows to configure a Dropbox storage backend. +type Config struct { + Token string + RemotePath string +} + +// NewStorageBackend creates and initializes a new Dropbox storage backend. +func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { + if opts.Token == "" { + return nil, errors.New("NewStorageBackend: No Dropbox token has been provided") + } else { + config := dropbox.Config{ + Token: opts.Token, + } + + client := files.New(config) + + return &dropboxStorage{ + StorageBackend: &storage.StorageBackend{ + DestinationPath: opts.RemotePath, + Log: logFunc, + }, + client: client, + }, nil + } +} + +// Name returns the name of the storage backend +func (b *dropboxStorage) Name() string { + return "Dropbox" +} + +// Copy copies the given file to the WebDav storage backend. +func (b *dropboxStorage) Copy(file string) error { + _, name := path.Split(file) + + folderArg := files.NewCreateFolderArg(b.DestinationPath) + if _, err := b.client.CreateFolderV2(folderArg); err != nil { + if err.(files.CreateFolderV2APIError).EndpointError.Path.Tag == files.WriteErrorConflict { + b.Log(storage.LogLevelInfo, b.Name(), "Destination path '%s' already exists in Dropbox, no new directory required.", b.DestinationPath) + } else { + return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s' in Dropbox: %w", b.DestinationPath, err) + } + } + + r, err := os.Open(file) + if err != nil { + return fmt.Errorf("(*dropboxStorage).Copy: Error opening the file to be uploaded: %w", err) + } + defer r.Close() + + // Start new upload session and get session id + + b.Log(storage.LogLevelInfo, b.Name(), "Starting upload session for backup '%s' to Dropbox at path '%s'.", file, b.DestinationPath) + + var sessionId string + uploadSessionStartArg := files.NewUploadSessionStartArg() + uploadSessionStartArg.SessionType = &files.UploadSessionType{Tagged: dropbox.Tagged{Tag: files.UploadSessionTypeConcurrent}} + if res, err := b.client.UploadSessionStart(uploadSessionStartArg, nil); err != nil { + return fmt.Errorf("(*dropboxStorage).Copy: Error starting the upload session: %w", err) + } else { + sessionId = res.SessionId + } + + // Send the file in 148MB chunks (Dropbox API limit is 150MB, concurrent upload requires a multiple of 4MB though) + // Last append can be any size <= 150MB with Close=True + + const chunkSize = 148 * 1024 * 1024 // 148MB + var offset uint64 = 0 + + for { + chunk := make([]byte, chunkSize) + bytesRead, err := r.Read(chunk) + if err != nil { + return fmt.Errorf("(*dropboxStorage).Copy: Error reading the file to be uploaded: %w", err) + } + chunk = chunk[:bytesRead] + + uploadSessionAppendArg := files.NewUploadSessionAppendArg( + files.NewUploadSessionCursor(sessionId, offset), + ) + isEOF := bytesRead < chunkSize + uploadSessionAppendArg.Close = isEOF + + if err := b.client.UploadSessionAppendV2(uploadSessionAppendArg, bytes.NewReader(chunk)); err != nil { + return fmt.Errorf("(*dropboxStorage).Copy: Error appending the file to the upload session: %w", err) + } + + if isEOF { + break + } + + offset += uint64(bytesRead) + } + + // Finish the upload session, commit the file (no new data added) + + b.client.UploadSessionFinish( + files.NewUploadSessionFinishArg( + files.NewUploadSessionCursor(sessionId, 0), + files.NewCommitInfo(filepath.Join(b.DestinationPath, name)), + ), nil) + + b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup '%s' to Dropbox at path '%s'.", file, b.DestinationPath) + + return nil +} + +// Prune rotates away backups according to the configuration and provided deadline for the Dropbox storage backend. +func (b *dropboxStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { + var entries []files.IsMetadata + res, err := b.client.ListFolder(files.NewListFolderArg(b.DestinationPath)) + if err != nil { + return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err) + } + entries = append(entries, res.Entries...) + + for res.HasMore { + res, err = b.client.ListFolderContinue(files.NewListFolderContinueArg(res.Cursor)) + if err != nil { + return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err) + } + entries = append(entries, res.Entries...) + } + + var matches []*files.FileMetadata + var lenCandidates int + for _, candidate := range entries { + if reflect.Indirect(reflect.ValueOf(candidate)).Type() != reflect.TypeOf(files.FileMetadata{}) { + continue + } + candidate := candidate.(*files.FileMetadata) + if !strings.HasPrefix(candidate.Name, pruningPrefix) { + continue + } + lenCandidates++ + if candidate.ServerModified.Before(deadline) { + matches = append(matches, candidate) + } + } + + stats := &storage.PruneStats{ + Total: uint(lenCandidates), + Pruned: uint(len(matches)), + } + + if err := b.DoPrune(b.Name(), len(matches), lenCandidates, "Dropbox backup(s)", func() error { + for _, match := range matches { + if _, err := b.client.DeleteV2(files.NewDeleteArg(filepath.Join(b.DestinationPath, match.Name))); err != nil { + return fmt.Errorf("(*dropboxStorage).Prune: Error removing file from Dropbox storage: %w", err) + } + } + return nil + }); err != nil { + return stats, err + } + + return stats, nil +}