From 336e12f8747c80a25a49e639416e3f5c1120afa8 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Fri, 9 Aug 2024 15:37:27 +0200 Subject: [PATCH] Add support for Azure storage access tiers (#452) --- cmd/backup/config.go | 1 + cmd/backup/script.go | 1 + docs/reference/index.md | 5 +++++ internal/storage/azure/azure.go | 32 +++++++++++++++++++++++++++----- test/azure/docker-compose.yml | 1 + 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 9c15ed7..0fb16de 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -76,6 +76,7 @@ type Config struct { AzureStorageContainerName string `split_words:"true"` AzureStoragePath string `split_words:"true"` AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` + AzureStorageAccessTier string `split_words:"true"` DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"` DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"` DropboxRefreshToken string `split_words:"true"` diff --git a/cmd/backup/script.go b/cmd/backup/script.go index e1f29da..d947e11 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -194,6 +194,7 @@ func (s *script) init() error { Endpoint: s.c.AzureStorageEndpoint, RemotePath: s.c.AzureStoragePath, ConnectionString: s.c.AzureStorageConnectionString, + AccessTier: s.c.AzureStorageAccessTier, } azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) if err != nil { diff --git a/docs/reference/index.md b/docs/reference/index.md index 4a0a86e..ffd76ce 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -269,6 +269,11 @@ You can populate below template according to your requirements and use it as you # Note: Use your app's subpath in Dropbox, if it doesn't have global access. # Consulte the README for further information. +# The access tier when using Azure Blob Storage. Possible values are +# https://github.com/Azure/azure-sdk-for-go/blob/sdk/storage/azblob/v1.3.2/sdk/storage/azblob/internal/generated/zz_constants.go#L14-L30 + +# AZURE_STORAGE_ACCESS_TIER="Cold" + # DROPBOX_REMOTE_PATH="/my/directory" # Number of concurrent chunked uploads for Dropbox. diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index 44631eb..3e0d88a 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -17,6 +17,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/offen/docker-volume-backup/internal/errwrap" "github.com/offen/docker-volume-backup/internal/storage" @@ -24,8 +26,9 @@ import ( type azureBlobStorage struct { *storage.StorageBackend - client *azblob.Client - containerName string + client *azblob.Client + uploadStreamOptions *blockblob.UploadStreamOptions + containerName string } // Config contains values that define the configuration of an Azure Blob Storage. @@ -36,6 +39,7 @@ type Config struct { ConnectionString string Endpoint string RemotePath string + AccessTier string } // NewStorageBackend creates and initializes a new Azure Blob Storage backend. @@ -81,9 +85,26 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error } } + var uploadStreamOptions *blockblob.UploadStreamOptions + if opts.AccessTier != "" { + var found bool + for _, t := range blob.PossibleAccessTierValues() { + if string(t) == opts.AccessTier { + found = true + uploadStreamOptions = &blockblob.UploadStreamOptions{ + AccessTier: &t, + } + } + } + if !found { + return nil, errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", opts.AccessTier)) + } + } + storage := azureBlobStorage{ - client: client, - containerName: opts.ContainerName, + client: client, + uploadStreamOptions: uploadStreamOptions, + containerName: opts.ContainerName, StorageBackend: &storage.StorageBackend{ DestinationPath: opts.RemotePath, Log: logFunc, @@ -103,12 +124,13 @@ func (b *azureBlobStorage) Copy(file string) error { if err != nil { return errwrap.Wrap(err, fmt.Sprintf("error opening file %s", file)) } + _, err = b.client.UploadStream( context.Background(), b.containerName, filepath.Join(b.DestinationPath, filepath.Base(file)), fileReader, - nil, + b.uploadStreamOptions, ) if err != nil { return errwrap.Wrap(err, fmt.Sprintf("error uploading file %s", file)) diff --git a/test/azure/docker-compose.yml b/test/azure/docker-compose.yml index e34e286..1fa1353 100644 --- a/test/azure/docker-compose.yml +++ b/test/azure/docker-compose.yml @@ -34,6 +34,7 @@ services: AZURE_STORAGE_CONTAINER_NAME: test-container AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/ AZURE_STORAGE_PATH: 'path/to/backup' + AZURE_STORAGE_ACCESS_TIER: Hot BACKUP_FILENAME: test.tar.gz BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}