Run copying and pruning against multiple storages in parallel (#152)

This commit is contained in:
Frederik Ring 2022-09-01 14:38:04 +02:00 committed by GitHub
parent 695a94d479
commit a6ec128cab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 26 additions and 9 deletions

View File

@ -32,6 +32,7 @@ import (
"github.com/otiai10/copy" "github.com/otiai10/copy"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp"
"golang.org/x/sync/errgroup"
) )
// script holds all the stateful information required to orchestrate a // script holds all the stateful information required to orchestrate a
@ -493,10 +494,15 @@ func (s *script) copyArchive() error {
} }
} }
eg := errgroup.Group{}
for _, backend := range s.storages { for _, backend := range s.storages {
if err := backend.Copy(s.file); err != nil { b := backend
return err eg.Go(func() error {
return b.Copy(s.file)
})
} }
if err := eg.Wait(); err != nil {
return fmt.Errorf("copyArchive: error copying archive: %w", err)
} }
return nil return nil
@ -512,16 +518,26 @@ func (s *script) pruneBackups() error {
deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway) deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway)
eg := errgroup.Group{}
for _, backend := range s.storages { for _, backend := range s.storages {
if stats, err := backend.Prune(deadline, s.c.BackupPruningPrefix); err == nil { b := backend
s.stats.Storages[backend.Name()] = StorageStats{ eg.Go(func() error {
stats, err := b.Prune(deadline, s.c.BackupPruningPrefix)
if err != nil {
return err
}
s.stats.Lock()
s.stats.Storages[b.Name()] = StorageStats{
Total: stats.Total, Total: stats.Total,
Pruned: stats.Pruned, Pruned: stats.Pruned,
} }
s.stats.Unlock()
} else { return nil
return err })
} }
if err := eg.Wait(); err != nil {
return fmt.Errorf("pruneBackups: error pruning backups: %w", err)
} }
return nil return nil

View File

@ -5,6 +5,7 @@ package main
import ( import (
"bytes" "bytes"
"sync"
"time" "time"
) )
@ -32,6 +33,7 @@ type StorageStats struct {
// Stats global stats regarding script execution // Stats global stats regarding script execution
type Stats struct { type Stats struct {
sync.Mutex
StartTime time.Time StartTime time.Time
EndTime time.Time EndTime time.Time
TookTime time.Duration TookTime time.Duration

View File

@ -35,7 +35,6 @@ type Config struct {
// NewStorageBackend creates and initializes a new WebDav storage backend. // NewStorageBackend creates and initializes a new WebDav storage backend.
func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
if opts.Username == "" || opts.Password == "" { if opts.Username == "" || opts.Password == "" {
return nil, errors.New("NewStorageBackend: WEBDAV_URL is defined, but no credentials were provided") return nil, errors.New("NewStorageBackend: WEBDAV_URL is defined, but no credentials were provided")
} else { } else {