offen/docker-volume-backup
2
0
Fork 0
mirror of https://github.com/offen/docker-volume-backup.git synced 2024-11-22 13:20:29 +01:00
Code Issues Packages Projects Releases Wiki Activity

Handle configuration errors for Azure storage upfront

Browse Source
This commit is contained in:
Frederik Ring 2024-08-11 15:08:06 +02:00
parent 8a64da4b0b
commit d8aa6db3f5
4 changed files with 115 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cmd/backup/command.go
View File

@ -36,6 +36,9 @@ func (c *command) runAsCommand() error {
} }
for _, config := range configurations { for _, config := range configurations {
if err := config.validate(); err != nil {
return errwrap.Wrap(err, "error validating config")
}
if err := runScript(config); err != nil { if err := runScript(config); err != nil {
return errwrap.Wrap(err, "error running script") return errwrap.Wrap(err, "error running script")
} }
@ -101,6 +104,12 @@ func (c *command) schedule(strategy configStrategy) error {
} }
for _, cfg := range configurations { for _, cfg := range configurations {
if err := cfg.validate(); err != nil {
return errwrap.Wrap(
err,
fmt.Sprintf("error validating config for schedule %s", cfg.BackupCronExpression),
)
}
config := cfg config := cfg
id, err := c.cr.AddFunc(config.BackupCronExpression, func() { id, err := c.cr.AddFunc(config.BackupCronExpression, func() {
c.logger.Info( c.logger.Info(

166
cmd/backup/config.go
View File

@ -12,83 +12,91 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/offen/docker-volume-backup/internal/errwrap" "github.com/offen/docker-volume-backup/internal/errwrap"
) )
// Config holds all configuration values that are expected to be set // Config holds all configuration values that are expected to be set
// by users. // by users.
type Config struct { type Config struct {
AwsS3BucketName string `split_words:"true"` AwsS3BucketName string `split_words:"true"`
AwsS3Path string `split_words:"true"` AwsS3Path string `split_words:"true"`
AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"`
AwsEndpointProto string `split_words:"true" default:"https"` AwsEndpointProto string `split_words:"true" default:"https"`
AwsEndpointInsecure bool `split_words:"true"` AwsEndpointInsecure bool `split_words:"true"`
AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"`
AwsStorageClass string `split_words:"true"` AwsStorageClass string `split_words:"true"`
AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"`
AwsSecretAccessKey string `split_words:"true"` AwsSecretAccessKey string `split_words:"true"`
AwsIamRoleEndpoint string `split_words:"true"` AwsIamRoleEndpoint string `split_words:"true"`
AwsPartSize int64 `split_words:"true"` AwsPartSize int64 `split_words:"true"`
BackupCompression CompressionType `split_words:"true" default:"gz"` BackupCompression CompressionType `split_words:"true" default:"gz"`
GzipParallelism WholeNumber `split_words:"true" default:"1"` GzipParallelism WholeNumber `split_words:"true" default:"1"`
BackupSources string `split_words:"true" default:"/backup"` BackupSources string `split_words:"true" default:"/backup"`
BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"` BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.{{ .Extension }}"`
BackupFilenameExpand bool `split_words:"true"` BackupFilenameExpand bool `split_words:"true"`
BackupLatestSymlink string `split_words:"true"` BackupLatestSymlink string `split_words:"true"`
BackupArchive string `split_words:"true" default:"/archive"` BackupArchive string `split_words:"true" default:"/archive"`
BackupCronExpression string `split_words:"true" default:"@daily"` BackupCronExpression string `split_words:"true" default:"@daily"`
BackupRetentionDays int32 `split_words:"true" default:"-1"` BackupRetentionDays int32 `split_words:"true" default:"-1"`
BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` BackupPruningLeeway time.Duration `split_words:"true" default:"1m"`
BackupPruningPrefix string `split_words:"true"` BackupPruningPrefix string `split_words:"true"`
BackupStopContainerLabel string `split_words:"true"` BackupStopContainerLabel string `split_words:"true"`
BackupStopDuringBackupLabel string `split_words:"true" default:"true"` BackupStopDuringBackupLabel string `split_words:"true" default:"true"`
BackupStopServiceTimeout time.Duration `split_words:"true" default:"5m"` BackupStopServiceTimeout time.Duration `split_words:"true" default:"5m"`
BackupFromSnapshot bool `split_words:"true"` BackupFromSnapshot bool `split_words:"true"`
BackupExcludeRegexp RegexpDecoder `split_words:"true"` BackupExcludeRegexp RegexpDecoder `split_words:"true"`
BackupSkipBackendsFromPrune []string `split_words:"true"` BackupSkipBackendsFromPrune []string `split_words:"true"`
GpgPassphrase string `split_words:"true"` GpgPassphrase string `split_words:"true"`
GpgPublicKeyRing string `split_words:"true"` GpgPublicKeyRing string `split_words:"true"`
NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` NotificationURLs []string `envconfig:"NOTIFICATION_URLS"`
NotificationLevel string `split_words:"true" default:"error"` NotificationLevel string `split_words:"true" default:"error"`
EmailNotificationRecipient string `split_words:"true"` EmailNotificationRecipient string `split_words:"true"`
EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` EmailNotificationSender string `split_words:"true" default:"noreply@nohost"`
EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"`
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
WebdavUrl string `split_words:"true"` WebdavUrl string `split_words:"true"`
WebdavUrlInsecure bool `split_words:"true"` WebdavUrlInsecure bool `split_words:"true"`
WebdavPath string `split_words:"true" default:"/"` WebdavPath string `split_words:"true" default:"/"`
WebdavUsername string `split_words:"true"` WebdavUsername string `split_words:"true"`
WebdavPassword string `split_words:"true"` WebdavPassword string `split_words:"true"`
SSHHostName string `split_words:"true"` SSHHostName string `split_words:"true"`
SSHPort string `split_words:"true" default:"22"` SSHPort string `split_words:"true" default:"22"`
SSHUser string `split_words:"true"` SSHUser string `split_words:"true"`
SSHPassword string `split_words:"true"` SSHPassword string `split_words:"true"`
SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"` SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"`
SSHIdentityPassphrase string `split_words:"true"` SSHIdentityPassphrase string `split_words:"true"`
SSHRemotePath string `split_words:"true"` SSHRemotePath string `split_words:"true"`
ExecLabel string `split_words:"true"` ExecLabel string `split_words:"true"`
ExecForwardOutput bool `split_words:"true"` ExecForwardOutput bool `split_words:"true"`
LockTimeout time.Duration `split_words:"true" default:"60m"` LockTimeout time.Duration `split_words:"true" default:"60m"`
AzureStorageAccountName string `split_words:"true"` AzureStorageAccountName string `split_words:"true"`
AzureStoragePrimaryAccountKey string `split_words:"true"` AzureStoragePrimaryAccountKey string `split_words:"true"`
AzureStorageConnectionString string `split_words:"true"` AzureStorageConnectionString string `split_words:"true"`
AzureStorageContainerName string `split_words:"true"` AzureStorageContainerName string `split_words:"true"`
AzureStoragePath string `split_words:"true"` AzureStoragePath string `split_words:"true"`
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
AzureStorageAccessTier string `split_words:"true"` AzureStorageAccessTier AzureStorageAccessTier `split_words:"true"`
DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"` DropboxEndpoint string `split_words:"true" default:"https://api.dropbox.com/"`
DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"` DropboxOAuth2Endpoint string `envconfig:"DROPBOX_OAUTH2_ENDPOINT" default:"https://api.dropbox.com/"`
DropboxRefreshToken string `split_words:"true"` DropboxRefreshToken string `split_words:"true"`
DropboxAppKey string `split_words:"true"` DropboxAppKey string `split_words:"true"`
DropboxAppSecret string `split_words:"true"` DropboxAppSecret string `split_words:"true"`
DropboxRemotePath string `split_words:"true"` DropboxRemotePath string `split_words:"true"`
DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"` DropboxConcurrencyLevel NaturalNumber `split_words:"true" default:"6"`
source string source string
additionalEnvVars map[string]string additionalEnvVars map[string]string
} }
func (c *Config) validate() error {
if c.AzureStoragePrimaryAccountKey != "" && c.AzureStorageConnectionString != "" {
return errwrap.Wrap(nil, "using azure primary account key and connection string are mutually exclusive")
}
return nil
}
type CompressionType string type CompressionType string
func (c *CompressionType) Decode(v string) error { func (c *CompressionType) Decode(v string) error {
@ -180,6 +188,30 @@ func (n *WholeNumber) Int() int {
return int(*n) return int(*n)
} }
type AzureStorageAccessTier string
func (t *AzureStorageAccessTier) Decode(v string) error {
if v == "" {
*t = ""
return nil
}
for _, a := range blob.PossibleAccessTierValues() {
if string(a) == v {
*t = AzureStorageAccessTier(v)
return nil
}
}
return errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", v))
}
func (t *AzureStorageAccessTier) AccessTier() *blob.AccessTier {
if *t == "" {
return nil
}
a := blob.AccessTier(*t)
return &a
}
type envVarLookup struct { type envVarLookup struct {
ok bool ok bool
key string key string

2
cmd/backup/script.go
View File

@ -199,7 +199,7 @@ func (s *script) init() error {
Endpoint: s.c.AzureStorageEndpoint, Endpoint: s.c.AzureStorageEndpoint,
RemotePath: s.c.AzureStoragePath, RemotePath: s.c.AzureStoragePath,
ConnectionString: s.c.AzureStorageConnectionString, ConnectionString: s.c.AzureStorageConnectionString,
AccessTier: s.c.AzureStorageAccessTier, AccessTier: s.c.AzureStorageAccessTier.AccessTier(),
} }
azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
if err != nil { if err != nil {

30
internal/storage/azure/azure.go
View File

@ -39,15 +39,11 @@ type Config struct {
ConnectionString string ConnectionString string
Endpoint string Endpoint string
RemotePath string RemotePath string
AccessTier string AccessTier *blob.AccessTier
} }
// NewStorageBackend creates and initializes a new Azure Blob Storage backend. // NewStorageBackend creates and initializes a new Azure Blob Storage backend.
func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
if opts.PrimaryAccountKey != "" && opts.ConnectionString != "" {
return nil, errwrap.Wrap(nil, "using primary account key and connection string are mutually exclusive")
}
endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint) endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint)
if err != nil { if err != nil {
return nil, errwrap.Wrap(err, "error parsing endpoint template") return nil, errwrap.Wrap(err, "error parsing endpoint template")
@ -85,26 +81,12 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error
} }
} }
var uploadStreamOptions *blockblob.UploadStreamOptions
if opts.AccessTier != "" {
var found bool
for _, t := range blob.PossibleAccessTierValues() {
if string(t) == opts.AccessTier {
found = true
uploadStreamOptions = &blockblob.UploadStreamOptions{
AccessTier: &t,
}
}
}
if !found {
return nil, errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", opts.AccessTier))
}
}
storage := azureBlobStorage{ storage := azureBlobStorage{
client: client, client: client,
uploadStreamOptions: uploadStreamOptions, uploadStreamOptions: &blockblob.UploadStreamOptions{
containerName: opts.ContainerName, AccessTier: opts.AccessTier,
},
containerName: opts.ContainerName,
StorageBackend: &storage.StorageBackend{ StorageBackend: &storage.StorageBackend{
DestinationPath: opts.RemotePath, DestinationPath: opts.RemotePath,
Log: logFunc, Log: logFunc,