offen/docker-volume-backup
2
0
Fork 0
mirror of https://github.com/offen/docker-volume-backup.git synced 2024-11-24 22:20:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fixed some instabilites. Changed default concurrency to 6.

Browse Source
This commit is contained in:
MaxJa4 2023-08-21 14:33:37 +02:00
parent 05c7a15e5b
commit f29b4c0994
2 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/backup/config.go
View File

@ -72,7 +72,7 @@ type Config struct {
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
DropboxToken string `split_words:"true"` DropboxToken string `split_words:"true"`
DropboxRemotePath string `split_words:"true"` DropboxRemotePath string `split_words:"true"`
DropboxConcurrencyLevel int `split_words:"true" default:"2"` DropboxConcurrencyLevel int `split_words:"true" default:"6"`
} }
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {

23
internal/storage/dropbox/dropbox.go
View File

@ -63,9 +63,14 @@ func (b *dropboxStorage) Copy(file string) error {
folderArg := files.NewCreateFolderArg(b.DestinationPath) folderArg := files.NewCreateFolderArg(b.DestinationPath)
if _, err := b.client.CreateFolderV2(folderArg); err != nil { if _, err := b.client.CreateFolderV2(folderArg); err != nil {
if err.(files.CreateFolderV2APIError).EndpointError.Path.Tag == files.WriteErrorConflict { switch err := err.(type) {
b.Log(storage.LogLevelInfo, b.Name(), "Destination path '%s' already exists in Dropbox, no new directory required.", b.DestinationPath) case files.CreateFolderV2APIError:
} else { if err.EndpointError.Path.Tag == files.WriteErrorConflict {
b.Log(storage.LogLevelInfo, b.Name(), "Destination path '%s' already exists in Dropbox, no new directory required.", b.DestinationPath)
} else {
return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s' in Dropbox: %w", b.DestinationPath, err)
}
default:
return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s' in Dropbox: %w", b.DestinationPath, err) return fmt.Errorf("(*dropboxStorage).Copy: Error creating directory '%s' in Dropbox: %w", b.DestinationPath, err)
} }
} }
@ -98,6 +103,7 @@ func (b *dropboxStorage) Copy(file string) error {
var errorChn = make(chan error, b.concurrencyLevel) var errorChn = make(chan error, b.concurrencyLevel)
var EOFChn = make(chan bool, b.concurrencyLevel) var EOFChn = make(chan bool, b.concurrencyLevel)
var mu sync.Mutex var mu sync.Mutex
var wg sync.WaitGroup
loop: loop:
for { for {
@ -106,12 +112,17 @@ loop:
case err := <-errorChn: // error from goroutine case err := <-errorChn: // error from goroutine
return err return err
case <-EOFChn: // EOF from goroutine case <-EOFChn: // EOF from goroutine
wg.Wait() // wait for all goroutines to finish
break loop break loop
default: default:
} }
go func() { go func() {
defer func() { <-guard }() defer func() {
wg.Done()
<-guard
}()
wg.Add(1)
chunk := make([]byte, chunkSize) chunk := make([]byte, chunkSize)
mu.Lock() // to preserve offset of chunks mu.Lock() // to preserve offset of chunks
@ -119,13 +130,15 @@ loop:
select { select {
case <-EOFChn: case <-EOFChn:
EOFChn <- true // put it back for outer loop EOFChn <- true // put it back for outer loop
return // already EOF mu.Unlock()
return // already EOF
default: default:
} }
bytesRead, err := r.Read(chunk) bytesRead, err := r.Read(chunk)
if err != nil { if err != nil {
errorChn <- fmt.Errorf("(*dropboxStorage).Copy: Error reading the file to be uploaded: %w", err) errorChn <- fmt.Errorf("(*dropboxStorage).Copy: Error reading the file to be uploaded: %w", err)
mu.Unlock()
return return
} }
chunk = chunk[:bytesRead] chunk = chunk[:bytesRead]