Implement pruning for Azure blob storage

cmd/backup/config.go

@@ -66,7 +66,7 @@ type Config struct {
 	AzureStorageAccountName       string        `split_words:"true"`
 	AzureStoragePrimaryAccountKey string        `split_words:"true"`
 	AzureStorageContainerName     string        `split_words:"true"`
-	AzureStorageEndpoint          string        `split_words:"true" default:"https://%s.blob.core.windows.net/"`
+	AzureStorageEndpoint          string        `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
 }
 
 func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {

internal/storage/azure/azure.go

@@ -4,14 +4,19 @@
 package azure
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os"
 	"path"
+	"sync"
+	"text/template"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/offen/docker-volume-backup/internal/storage"
+	"github.com/offen/docker-volume-backup/internal/utilities"
 )
 
 type azureBlobStorage struct {
@@ -34,13 +39,26 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error
 	if err != nil {
 		return nil, fmt.Errorf("NewStorageBackend: error creating shared Azure credential: %w", err)
 	}
-	client, err := azblob.NewClientWithSharedKeyCredential(fmt.Sprintf(opts.Endpoint, opts.AccountName), cred, nil)
+
+	endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint)
+	if err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error parsing endpoint template: %w", err)
+	}
+
+	var ep bytes.Buffer
+	if err := endpointTemplate.Execute(&ep, opts); err != nil {
+		return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err)
+	}
+	client, err := azblob.NewClientWithSharedKeyCredential(ep.String(), cred, nil)
 	if err != nil {
 		return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err)
 	}
 	storage := azureBlobStorage{
 		client:        client,
 		containerName: opts.ContainerName,
+		StorageBackend: &storage.StorageBackend{
+			Log: logFunc,
+		},
 	}
 	return &storage, nil
 }
@@ -57,7 +75,8 @@ func (b *azureBlobStorage) Copy(file string) error {
 		return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err)
 	}
 
-	_, err = b.client.UploadStream(context.TODO(),
+	_, err = b.client.UploadStream(
+		context.Background(),
 		b.containerName,
 		path.Base(file),
 		fileReader,
@@ -69,7 +88,55 @@ func (b *azureBlobStorage) Copy(file string) error {
 	return nil
 }
 
-// Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend.
+// Prune rotates away backups according to the configuration and provided
+// deadline for the Azure Blob storage backend.
 func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
-	return &storage.PruneStats{}, nil
+	pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{
+		Prefix: &pruningPrefix,
+	})
+	var matches []string
+	var totalCount uint
+	for pager.More() {
+		resp, err := pager.NextPage(context.Background())
+		if err != nil {
+			return nil, fmt.Errorf("(*azureBlobStorage).Prune: error paging over blobs: %w", err)
+		}
+		for _, v := range resp.Segment.BlobItems {
+			totalCount++
+			if v.Properties.LastModified.Before(deadline) {
+				matches = append(matches, *v.Name)
+			}
+		}
+	}
+
+	stats := storage.PruneStats{
+		Total:  totalCount,
+		Pruned: uint(len(matches)),
+	}
+
+	if err := b.DoPrune(b.Name(), len(matches), int(totalCount), "Azure Blob Storage backup(s)", func() error {
+		wg := sync.WaitGroup{}
+		wg.Add(len(matches))
+		var errors []error
+
+		for _, match := range matches {
+			name := match
+			go func() {
+				_, err := b.client.DeleteBlob(context.Background(), b.containerName, name, nil)
+				if err != nil {
+					errors = append(errors, err)
+				}
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+		if len(errors) != 0 {
+			return utilities.Join(errors...)
+		}
+		return nil
+	}); err != nil {
+		return &stats, err
+	}
+
+	return &stats, nil
 }

test/azure/docker-compose.yml

@@ -34,7 +34,7 @@ services:
       AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1
       AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
       AZURE_STORAGE_CONTAINER_NAME: test-container
-      AZURE_STORAGE_ENDPOINT: http://storage:10000/%s/
+      AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/
       BACKUP_FILENAME: test.tar.gz
       BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
       BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}