From d8aa6db3f5fd7cf3fef77cfa9e7ad01cdbe8e247 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Sun, 11 Aug 2024 15:08:06 +0200 Subject: [PATCH] Handle configuration errors for Azure storage upfront --- cmd/backup/command.go | 9 ++ cmd/backup/config.go | 166 +++++++++++++++++++------------- cmd/backup/script.go | 2 +- internal/storage/azure/azure.go | 30 ++---- 4 files changed, 115 insertions(+), 92 deletions(-) diff --git a/cmd/backup/command.go b/cmd/backup/command.go index 10fa0e1..61ae3f5 100644 --- a/cmd/backup/command.go +++ b/cmd/backup/command.go @@ -36,6 +36,9 @@ func (c *command) runAsCommand() error { } for _, config := range configurations { + if err := config.validate(); err != nil { + return errwrap.Wrap(err, "error validating config") + } if err := runScript(config); err != nil { return errwrap.Wrap(err, "error running script") } @@ -101,6 +104,12 @@ func (c *command) schedule(strategy configStrategy) error { } for _, cfg := range configurations { + if err := cfg.validate(); err != nil { + return errwrap.Wrap( + err, + fmt.Sprintf("error validating config for schedule %s", cfg.BackupCronExpression), + ) + } config := cfg id, err := c.cr.AddFunc(config.BackupCronExpression, func() { c.logger.Info( diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 4230827..b0ca3c1 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -12,83 +12,91 @@ import ( "strconv" "time" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/offen/docker-volume-backup/internal/errwrap" ) // Config holds all configuration values that are expected to be set // by users. type Config struct { - AwsS3BucketName string `split_words:"true"` - AwsS3Path string `split_words:"true"` - AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` - AwsEndpointProto string `split_words:"true" default:"https"` - AwsEndpointInsecure bool `split_words:"true"` - AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` - AwsStorageClass string `split_words:"true"` - AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` - AwsSecretAccessKey string `split_words:"true"` - AwsIamRoleEndpoint string `split_words:"true"` - AwsPartSize int64 `split_words:"true"` - BackupCompression CompressionType `split_words:"true" default:"gz"` - GzipParallelism WholeNumber `split_words:"true" default:"1"` - BackupSources string `split_words:"true" default:"/backup"` - BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"` - BackupFilenameExpand bool `split_words:"true"` - BackupLatestSymlink string `split_words:"true"` - BackupArchive string `split_words:"true" default:"/archive"` - BackupCronExpression string `split_words:"true" default:"@daily"` - BackupRetentionDays int32 `split_words:"true" default:"-1"` - BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` - BackupPruningPrefix string `split_words:"true"` - BackupStopContainerLabel string `split_words:"true"` - BackupStopDuringBackupLabel string `split_words:"true" default:"true"` - BackupStopServiceTimeout time.Duration `split_words:"true" default:"5m"` - BackupFromSnapshot bool `split_words:"true"` - BackupExcludeRegexp RegexpDecoder `split_words:"true"` - BackupSkipBackendsFromPrune []string `split_words:"true"` - GpgPassphrase string `split_words:"true"` - GpgPublicKeyRing string `split_words:"true"` - NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` - NotificationLevel string `split_words:"true" default:"error"` - EmailNotificationRecipient string `split_words:"true"` - EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` - EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` - EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` - EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` - EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` - WebdavUrl string `split_words:"true"` - WebdavUrlInsecure bool `split_words:"true"` - WebdavPath string `split_words:"true" default:"/"` - WebdavUsername string `split_words:"true"` - WebdavPassword string `split_words:"true"` - SSHHostName string `split_words:"true"` - SSHPort string `split_words:"true" default:"22"` - SSHUser string `split_words:"true"` - SSHPassword string `split_words:"true"` - SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"` - SSHIdentityPassphrase string `split_words:"true"` - SSHRemotePath string `split_words:"true"` - ExecLabel string `split_words:"true"` - ExecForwardOutput bool `split_words:"true"` - LockTimeout time.Duration `split_words:"true" default:"60m"` - AzureStorageAccountName string `split_words:"true"` - AzureStoragePrimaryAccountKey string `split_words:"true"` - AzureStorageConnectionString string `split_words:"true"` - AzureStorageContainerName string `split_words:"true"` - AzureStoragePath string `split_words:"true"` - AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` - AzureStorageAccessTier string `split_words:"true"` - DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"` - DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"` - DropboxRefreshToken string `split_words:"true"` - DropboxAppKey string `split_words:"true"` - DropboxAppSecret string `split_words:"true"` - DropboxRemotePath string `split_words:"true"` - DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"` + AwsS3BucketName string `split_words:"true"` + AwsS3Path string `split_words:"true"` + AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` + AwsEndpointProto string `split_words:"true" default:"https"` + AwsEndpointInsecure bool `split_words:"true"` + AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` + AwsStorageClass string `split_words:"true"` + AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` + AwsSecretAccessKey string `split_words:"true"` + AwsIamRoleEndpoint string `split_words:"true"` + AwsPartSize int64 `split_words:"true"` + BackupCompression CompressionType `split_words:"true" default:"gz"` + GzipParallelism WholeNumber `split_words:"true" default:"1"` + BackupSources string `split_words:"true" default:"/backup"` + BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"` + BackupFilenameExpand bool `split_words:"true"` + BackupLatestSymlink string `split_words:"true"` + BackupArchive string `split_words:"true" default:"/archive"` + BackupCronExpression string `split_words:"true" default:"@daily"` + BackupRetentionDays int32 `split_words:"true" default:"-1"` + BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` + BackupPruningPrefix string `split_words:"true"` + BackupStopContainerLabel string `split_words:"true"` + BackupStopDuringBackupLabel string `split_words:"true" default:"true"` + BackupStopServiceTimeout time.Duration `split_words:"true" default:"5m"` + BackupFromSnapshot bool `split_words:"true"` + BackupExcludeRegexp RegexpDecoder `split_words:"true"` + BackupSkipBackendsFromPrune []string `split_words:"true"` + GpgPassphrase string `split_words:"true"` + GpgPublicKeyRing string `split_words:"true"` + NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` + NotificationLevel string `split_words:"true" default:"error"` + EmailNotificationRecipient string `split_words:"true"` + EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` + EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` + EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` + EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` + EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` + WebdavUrl string `split_words:"true"` + WebdavUrlInsecure bool `split_words:"true"` + WebdavPath string `split_words:"true" default:"/"` + WebdavUsername string `split_words:"true"` + WebdavPassword string `split_words:"true"` + SSHHostName string `split_words:"true"` + SSHPort string `split_words:"true" default:"22"` + SSHUser string `split_words:"true"` + SSHPassword string `split_words:"true"` + SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"` + SSHIdentityPassphrase string `split_words:"true"` + SSHRemotePath string `split_words:"true"` + ExecLabel string `split_words:"true"` + ExecForwardOutput bool `split_words:"true"` + LockTimeout time.Duration `split_words:"true" default:"60m"` + AzureStorageAccountName string `split_words:"true"` + AzureStoragePrimaryAccountKey string `split_words:"true"` + AzureStorageConnectionString string `split_words:"true"` + AzureStorageContainerName string `split_words:"true"` + AzureStoragePath string `split_words:"true"` + AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` + AzureStorageAccessTier AzureStorageAccessTier `split_words:"true"` + DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"` + DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"` + DropboxRefreshToken string `split_words:"true"` + DropboxAppKey string `split_words:"true"` + DropboxAppSecret string `split_words:"true"` + DropboxRemotePath string `split_words:"true"` + DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"` source string additionalEnvVars map[string]string } +func (c *Config) validate() error { + if c.AzureStoragePrimaryAccountKey != "" && c.AzureStorageConnectionString != "" { + return errwrap.Wrap(nil, "using azure primary account key and connection string are mutually exclusive") + } + return nil +} + type CompressionType string func (c *CompressionType) Decode(v string) error { @@ -180,6 +188,30 @@ func (n *WholeNumber) Int() int { return int(*n) } +type AzureStorageAccessTier string + +func (t *AzureStorageAccessTier) Decode(v string) error { + if v == "" { + *t = "" + return nil + } + for _, a := range blob.PossibleAccessTierValues() { + if string(a) == v { + *t = AzureStorageAccessTier(v) + return nil + } + } + return errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", v)) +} + +func (t *AzureStorageAccessTier) AccessTier() *blob.AccessTier { + if *t == "" { + return nil + } + a := blob.AccessTier(*t) + return &a +} + type envVarLookup struct { ok bool key string diff --git a/cmd/backup/script.go b/cmd/backup/script.go index 56b8af8..92a92b3 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -199,7 +199,7 @@ func (s *script) init() error { Endpoint: s.c.AzureStorageEndpoint, RemotePath: s.c.AzureStoragePath, ConnectionString: s.c.AzureStorageConnectionString, - AccessTier: s.c.AzureStorageAccessTier, + AccessTier: s.c.AzureStorageAccessTier.AccessTier(), } azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) if err != nil { diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go index 3e0d88a..dc61120 100644 --- a/internal/storage/azure/azure.go +++ b/internal/storage/azure/azure.go @@ -39,15 +39,11 @@ type Config struct { ConnectionString string Endpoint string RemotePath string - AccessTier string + AccessTier *blob.AccessTier } // NewStorageBackend creates and initializes a new Azure Blob Storage backend. func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { - if opts.PrimaryAccountKey != "" && opts.ConnectionString != "" { - return nil, errwrap.Wrap(nil, "using primary account key and connection string are mutually exclusive") - } - endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint) if err != nil { return nil, errwrap.Wrap(err, "error parsing endpoint template") @@ -85,26 +81,12 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error } } - var uploadStreamOptions *blockblob.UploadStreamOptions - if opts.AccessTier != "" { - var found bool - for _, t := range blob.PossibleAccessTierValues() { - if string(t) == opts.AccessTier { - found = true - uploadStreamOptions = &blockblob.UploadStreamOptions{ - AccessTier: &t, - } - } - } - if !found { - return nil, errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", opts.AccessTier)) - } - } - storage := azureBlobStorage{ - client: client, - uploadStreamOptions: uploadStreamOptions, - containerName: opts.ContainerName, + client: client, + uploadStreamOptions: &blockblob.UploadStreamOptions{ + AccessTier: opts.AccessTier, + }, + containerName: opts.ContainerName, StorageBackend: &storage.StorageBackend{ DestinationPath: opts.RemotePath, Log: logFunc,