diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 09ef3d2..398a96a 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -66,7 +66,7 @@ type Config struct { AzureStorageAccountName string `split_words:"true"` AzureStoragePrimaryAccountKey string `split_words:"true"` AzureStorageContainerName string `split_words:"true"` - AzureStorageEndpoint string `split_words:"true" default:"https://%s.blob.core.windows.net/"` + AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` } func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index 82165ce..dd850bb 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -4,14 +4,19 @@ package azure import ( + "bytes" "context" "fmt" "os" "path" + "sync" + "text/template" "time" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/offen/docker-volume-backup/internal/storage" + "github.com/offen/docker-volume-backup/internal/utilities" ) type azureBlobStorage struct { @@ -34,13 +39,26 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error if err != nil { return nil, fmt.Errorf("NewStorageBackend: error creating shared Azure credential: %w", err) } - client, err := azblob.NewClientWithSharedKeyCredential(fmt.Sprintf(opts.Endpoint, opts.AccountName), cred, nil) + + endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error parsing endpoint template: %w", err) + } + + var ep bytes.Buffer + if err := endpointTemplate.Execute(&ep, opts); err != nil { + return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err) + } + client, err := azblob.NewClientWithSharedKeyCredential(ep.String(), cred, nil) if err != nil { return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) } storage := azureBlobStorage{ client: client, containerName: opts.ContainerName, + StorageBackend: &storage.StorageBackend{ + Log: logFunc, + }, } return &storage, nil } @@ -57,7 +75,8 @@ func (b *azureBlobStorage) Copy(file string) error { return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err) } - _, err = b.client.UploadStream(context.TODO(), + _, err = b.client.UploadStream( + context.Background(), b.containerName, path.Base(file), fileReader, @@ -69,7 +88,55 @@ func (b *azureBlobStorage) Copy(file string) error { return nil } -// Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend. +// Prune rotates away backups according to the configuration and provided +// deadline for the Azure Blob storage backend. func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { - return &storage.PruneStats{}, nil + pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{ + Prefix: &pruningPrefix, + }) + var matches []string + var totalCount uint + for pager.More() { + resp, err := pager.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("(*azureBlobStorage).Prune: error paging over blobs: %w", err) + } + for _, v := range resp.Segment.BlobItems { + totalCount++ + if v.Properties.LastModified.Before(deadline) { + matches = append(matches, *v.Name) + } + } + } + + stats := storage.PruneStats{ + Total: totalCount, + Pruned: uint(len(matches)), + } + + if err := b.DoPrune(b.Name(), len(matches), int(totalCount), "Azure Blob Storage backup(s)", func() error { + wg := sync.WaitGroup{} + wg.Add(len(matches)) + var errors []error + + for _, match := range matches { + name := match + go func() { + _, err := b.client.DeleteBlob(context.Background(), b.containerName, name, nil) + if err != nil { + errors = append(errors, err) + } + wg.Done() + }() + } + wg.Wait() + if len(errors) != 0 { + return utilities.Join(errors...) + } + return nil + }); err != nil { + return &stats, err + } + + return &stats, nil } diff --git a/test/azure/docker-compose.yml b/test/azure/docker-compose.yml index 5e3ff01..a1146c3 100644 --- a/test/azure/docker-compose.yml +++ b/test/azure/docker-compose.yml @@ -34,7 +34,7 @@ services: AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1 AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== AZURE_STORAGE_CONTAINER_NAME: test-container - AZURE_STORAGE_ENDPOINT: http://storage:10000/%s/ + AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/ BACKUP_FILENAME: test.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}