mirror of
https://github.com/offen/docker-volume-backup.git
synced 2024-11-10 00:30:29 +01:00
Add new storage backend: Dropbox (#103)
This commit is contained in:
parent
67e7288855
commit
4e04cb9926
@ -70,6 +70,8 @@ type Config struct {
|
||||
AzureStorageContainerName string `split_words:"true"`
|
||||
AzureStoragePath string `split_words:"true"`
|
||||
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
|
||||
DropboxToken string `split_words:"true"`
|
||||
DropboxRemotePath string `split_words:"true"`
|
||||
}
|
||||
|
||||
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
|
||||
|
@ -19,6 +19,7 @@ import (
|
||||
|
||||
"github.com/offen/docker-volume-backup/internal/storage"
|
||||
"github.com/offen/docker-volume-backup/internal/storage/azure"
|
||||
"github.com/offen/docker-volume-backup/internal/storage/dropbox"
|
||||
"github.com/offen/docker-volume-backup/internal/storage/local"
|
||||
"github.com/offen/docker-volume-backup/internal/storage/s3"
|
||||
"github.com/offen/docker-volume-backup/internal/storage/ssh"
|
||||
@ -70,11 +71,12 @@ func newScript() (*script, error) {
|
||||
StartTime: time.Now(),
|
||||
LogOutput: logBuffer,
|
||||
Storages: map[string]StorageStats{
|
||||
"S3": {},
|
||||
"WebDAV": {},
|
||||
"SSH": {},
|
||||
"Local": {},
|
||||
"Azure": {},
|
||||
"S3": {},
|
||||
"WebDAV": {},
|
||||
"SSH": {},
|
||||
"Local": {},
|
||||
"Azure": {},
|
||||
"Dropbox": {},
|
||||
},
|
||||
},
|
||||
}
|
||||
@ -218,6 +220,18 @@ func newScript() (*script, error) {
|
||||
s.storages = append(s.storages, azureBackend)
|
||||
}
|
||||
|
||||
if s.c.DropboxToken != "" {
|
||||
dropboxConfig := dropbox.Config{
|
||||
Token: s.c.DropboxToken,
|
||||
RemotePath: s.c.DropboxRemotePath,
|
||||
}
|
||||
dropboxBackend, err := dropbox.NewStorageBackend(dropboxConfig, logFunc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.storages = append(s.storages, dropboxBackend)
|
||||
}
|
||||
|
||||
if s.c.EmailNotificationRecipient != "" {
|
||||
emailURL := fmt.Sprintf(
|
||||
"smtp://%s:%s@%s:%d/?from=%s&to=%s",
|
||||
|
8
go.mod
8
go.mod
@ -20,6 +20,13 @@ require (
|
||||
golang.org/x/sync v0.3.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/protobuf v1.28.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
|
||||
@ -28,6 +35,7 @@ require (
|
||||
github.com/docker/distribution v2.8.2+incompatible // indirect
|
||||
github.com/docker/go-connections v0.4.0 // indirect
|
||||
github.com/docker/go-units v0.4.0 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
|
4
go.sum
4
go.sum
@ -257,6 +257,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
|
||||
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
|
||||
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
|
||||
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU=
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
@ -785,6 +787,7 @@ golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7Lm
|
||||
golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
|
||||
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
|
||||
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
|
||||
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk=
|
||||
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -1046,6 +1049,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
182
internal/storage/dropbox/dropbox.go
Normal file
182
internal/storage/dropbox/dropbox.go
Normal file
@ -0,0 +1,182 @@
|
||||
package dropbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
|
||||
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
|
||||
"github.com/offen/docker-volume-backup/internal/storage"
|
||||
)
|
||||
|
||||
type dropboxStorage struct {
|
||||
*storage.StorageBackend
|
||||
client files.Client
|
||||
}
|
||||
|
||||
// Config allows to configure a Dropbox storage backend.
|
||||
type Config struct {
|
||||
Token string
|
||||
RemotePath string
|
||||
}
|
||||
|
||||
// NewStorageBackend creates and initializes a new Dropbox storage backend.
|
||||
func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
|
||||
if opts.Token == "" {
|
||||
return nil, errors.New("NewStorageBackend: No Dropbox token has been provided")
|
||||
} else {
|
||||
config := dropbox.Config{
|
||||
Token: opts.Token,
|
||||
}
|
||||
|
||||
client := files.New(config)
|
||||
|
||||
return &dropboxStorage{
|
||||
StorageBackend: &storage.StorageBackend{
|
||||
DestinationPath: opts.RemotePath,
|
||||
Log: logFunc,
|
||||
},
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the name of the storage backend
|
||||
func (b *dropboxStorage) Name() string {
|
||||
return "Dropbox"
|
||||
}
|
||||
|
||||
// Copy copies the given file to the WebDav storage backend.
|
||||
func (b *dropboxStorage) Copy(file string) error {
|
||||
_, name := path.Split(file)
|
||||
|
||||
folderArg := files.NewCreateFolderArg(b.DestinationPath)
|
||||
if _, err := b.client.CreateFolderV2(folderArg); err != nil {
|
||||
if err.(files.CreateFolderV2APIError).EndpointError.Path.Tag == files.WriteErrorConflict {
|
||||
b.Log(storage.LogLevelInfo, b.Name(), "Destination path '%s' already exists in Dropbox, no new directory required.", b.DestinationPath)
|
||||
} else {
|
||||
return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s' in Dropbox: %w", b.DestinationPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
r, err := os.Open(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("(*dropboxStorage).Copy: Error opening the file to be uploaded: %w", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
// Start new upload session and get session id
|
||||
|
||||
b.Log(storage.LogLevelInfo, b.Name(), "Starting upload session for backup '%s' to Dropbox at path '%s'.", file, b.DestinationPath)
|
||||
|
||||
var sessionId string
|
||||
uploadSessionStartArg := files.NewUploadSessionStartArg()
|
||||
uploadSessionStartArg.SessionType = &files.UploadSessionType{Tagged: dropbox.Tagged{Tag: files.UploadSessionTypeConcurrent}}
|
||||
if res, err := b.client.UploadSessionStart(uploadSessionStartArg, nil); err != nil {
|
||||
return fmt.Errorf("(*dropboxStorage).Copy: Error starting the upload session: %w", err)
|
||||
} else {
|
||||
sessionId = res.SessionId
|
||||
}
|
||||
|
||||
// Send the file in 148MB chunks (Dropbox API limit is 150MB, concurrent upload requires a multiple of 4MB though)
|
||||
// Last append can be any size <= 150MB with Close=True
|
||||
|
||||
const chunkSize = 148 * 1024 * 1024 // 148MB
|
||||
var offset uint64 = 0
|
||||
|
||||
for {
|
||||
chunk := make([]byte, chunkSize)
|
||||
bytesRead, err := r.Read(chunk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("(*dropboxStorage).Copy: Error reading the file to be uploaded: %w", err)
|
||||
}
|
||||
chunk = chunk[:bytesRead]
|
||||
|
||||
uploadSessionAppendArg := files.NewUploadSessionAppendArg(
|
||||
files.NewUploadSessionCursor(sessionId, offset),
|
||||
)
|
||||
isEOF := bytesRead < chunkSize
|
||||
uploadSessionAppendArg.Close = isEOF
|
||||
|
||||
if err := b.client.UploadSessionAppendV2(uploadSessionAppendArg, bytes.NewReader(chunk)); err != nil {
|
||||
return fmt.Errorf("(*dropboxStorage).Copy: Error appending the file to the upload session: %w", err)
|
||||
}
|
||||
|
||||
if isEOF {
|
||||
break
|
||||
}
|
||||
|
||||
offset += uint64(bytesRead)
|
||||
}
|
||||
|
||||
// Finish the upload session, commit the file (no new data added)
|
||||
|
||||
b.client.UploadSessionFinish(
|
||||
files.NewUploadSessionFinishArg(
|
||||
files.NewUploadSessionCursor(sessionId, 0),
|
||||
files.NewCommitInfo(filepath.Join(b.DestinationPath, name)),
|
||||
), nil)
|
||||
|
||||
b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup '%s' to Dropbox at path '%s'.", file, b.DestinationPath)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Prune rotates away backups according to the configuration and provided deadline for the Dropbox storage backend.
|
||||
func (b *dropboxStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
|
||||
var entries []files.IsMetadata
|
||||
res, err := b.client.ListFolder(files.NewListFolderArg(b.DestinationPath))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err)
|
||||
}
|
||||
entries = append(entries, res.Entries...)
|
||||
|
||||
for res.HasMore {
|
||||
res, err = b.client.ListFolderContinue(files.NewListFolderContinueArg(res.Cursor))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("(*webDavStorage).Prune: Error looking up candidates from remote storage: %w", err)
|
||||
}
|
||||
entries = append(entries, res.Entries...)
|
||||
}
|
||||
|
||||
var matches []*files.FileMetadata
|
||||
var lenCandidates int
|
||||
for _, candidate := range entries {
|
||||
if reflect.Indirect(reflect.ValueOf(candidate)).Type() != reflect.TypeOf(files.FileMetadata{}) {
|
||||
continue
|
||||
}
|
||||
candidate := candidate.(*files.FileMetadata)
|
||||
if !strings.HasPrefix(candidate.Name, pruningPrefix) {
|
||||
continue
|
||||
}
|
||||
lenCandidates++
|
||||
if candidate.ServerModified.Before(deadline) {
|
||||
matches = append(matches, candidate)
|
||||
}
|
||||
}
|
||||
|
||||
stats := &storage.PruneStats{
|
||||
Total: uint(lenCandidates),
|
||||
Pruned: uint(len(matches)),
|
||||
}
|
||||
|
||||
if err := b.DoPrune(b.Name(), len(matches), lenCandidates, "Dropbox backup(s)", func() error {
|
||||
for _, match := range matches {
|
||||
if _, err := b.client.DeleteV2(files.NewDeleteArg(filepath.Join(b.DestinationPath, match.Name))); err != nil {
|
||||
return fmt.Errorf("(*dropboxStorage).Prune: Error removing file from Dropbox storage: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
Loading…
Reference in New Issue
Block a user