Add support for Azure storage access tiers (#452)

This commit is contained in:
Frederik Ring 2024-08-09 15:37:27 +02:00 committed by GitHub
parent 016c6c8307
commit 336e12f874
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 35 additions and 5 deletions

View File

@ -76,6 +76,7 @@ type Config struct {
AzureStorageContainerName string `split_words:"true"` AzureStorageContainerName string `split_words:"true"`
AzureStoragePath string `split_words:"true"` AzureStoragePath string `split_words:"true"`
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
AzureStorageAccessTier string `split_words:"true"`
DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"` DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"`
DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"` DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"`
DropboxRefreshToken string `split_words:"true"` DropboxRefreshToken string `split_words:"true"`

View File

@ -194,6 +194,7 @@ func (s *script) init() error {
Endpoint: s.c.AzureStorageEndpoint, Endpoint: s.c.AzureStorageEndpoint,
RemotePath: s.c.AzureStoragePath, RemotePath: s.c.AzureStoragePath,
ConnectionString: s.c.AzureStorageConnectionString, ConnectionString: s.c.AzureStorageConnectionString,
AccessTier: s.c.AzureStorageAccessTier,
} }
azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
if err != nil { if err != nil {

View File

@ -269,6 +269,11 @@ You can populate below template according to your requirements and use it as you
# Note: Use your app's subpath in Dropbox, if it doesn't have global access. # Note: Use your app's subpath in Dropbox, if it doesn't have global access.
# Consulte the README for further information. # Consulte the README for further information.
# The access tier when using Azure Blob Storage. Possible values are
# https://github.com/Azure/azure-sdk-for-go/blob/sdk/storage/azblob/v1.3.2/sdk/storage/azblob/internal/generated/zz_constants.go#L14-L30
# AZURE_STORAGE_ACCESS_TIER="Cold"
# DROPBOX_REMOTE_PATH="/my/directory" # DROPBOX_REMOTE_PATH="/my/directory"
# Number of concurrent chunked uploads for Dropbox. # Number of concurrent chunked uploads for Dropbox.

View File

@ -17,6 +17,8 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/offen/docker-volume-backup/internal/errwrap" "github.com/offen/docker-volume-backup/internal/errwrap"
"github.com/offen/docker-volume-backup/internal/storage" "github.com/offen/docker-volume-backup/internal/storage"
@ -24,8 +26,9 @@ import (
type azureBlobStorage struct { type azureBlobStorage struct {
*storage.StorageBackend *storage.StorageBackend
client *azblob.Client client *azblob.Client
containerName string uploadStreamOptions *blockblob.UploadStreamOptions
containerName string
} }
// Config contains values that define the configuration of an Azure Blob Storage. // Config contains values that define the configuration of an Azure Blob Storage.
@ -36,6 +39,7 @@ type Config struct {
ConnectionString string ConnectionString string
Endpoint string Endpoint string
RemotePath string RemotePath string
AccessTier string
} }
// NewStorageBackend creates and initializes a new Azure Blob Storage backend. // NewStorageBackend creates and initializes a new Azure Blob Storage backend.
@ -81,9 +85,26 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error
} }
} }
var uploadStreamOptions *blockblob.UploadStreamOptions
if opts.AccessTier != "" {
var found bool
for _, t := range blob.PossibleAccessTierValues() {
if string(t) == opts.AccessTier {
found = true
uploadStreamOptions = &blockblob.UploadStreamOptions{
AccessTier: &t,
}
}
}
if !found {
return nil, errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", opts.AccessTier))
}
}
storage := azureBlobStorage{ storage := azureBlobStorage{
client: client, client: client,
containerName: opts.ContainerName, uploadStreamOptions: uploadStreamOptions,
containerName: opts.ContainerName,
StorageBackend: &storage.StorageBackend{ StorageBackend: &storage.StorageBackend{
DestinationPath: opts.RemotePath, DestinationPath: opts.RemotePath,
Log: logFunc, Log: logFunc,
@ -103,12 +124,13 @@ func (b *azureBlobStorage) Copy(file string) error {
if err != nil { if err != nil {
return errwrap.Wrap(err, fmt.Sprintf("error opening file %s", file)) return errwrap.Wrap(err, fmt.Sprintf("error opening file %s", file))
} }
_, err = b.client.UploadStream( _, err = b.client.UploadStream(
context.Background(), context.Background(),
b.containerName, b.containerName,
filepath.Join(b.DestinationPath, filepath.Base(file)), filepath.Join(b.DestinationPath, filepath.Base(file)),
fileReader, fileReader,
nil, b.uploadStreamOptions,
) )
if err != nil { if err != nil {
return errwrap.Wrap(err, fmt.Sprintf("error uploading file %s", file)) return errwrap.Wrap(err, fmt.Sprintf("error uploading file %s", file))

View File

@ -34,6 +34,7 @@ services:
AZURE_STORAGE_CONTAINER_NAME: test-container AZURE_STORAGE_CONTAINER_NAME: test-container
AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/ AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/
AZURE_STORAGE_PATH: 'path/to/backup' AZURE_STORAGE_PATH: 'path/to/backup'
AZURE_STORAGE_ACCESS_TIER: Hot
BACKUP_FILENAME: test.tar.gz BACKUP_FILENAME: test.tar.gz
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}