From 5c04e11f10557b2d2913a3d2e98d1204fe6e0543 Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Wed, 11 Jan 2023 21:40:48 +0100 Subject: [PATCH] Add support for Azure Blob Storage (#171) * Scaffold Azure storage backend that does nothing yet * Implement copy for Azure Blob Storage * Set up automated testing for Azure Storage * Implement pruning for Azure blob storage * Add documentation for Azure Blob Storage * Add support for remote path * Add azure to notifications doc * Tidy go.mod file * Allow use of managed identity credential * Use volume in tests * Auto append trailing slash to endpoint if needed, clarify docs, tidy mod file --- README.md | 43 ++++++++- cmd/backup/config.go | 99 ++++++++++---------- cmd/backup/script.go | 17 ++++ docs/NOTIFICATION-TEMPLATES.md | 2 +- go.mod | 8 ++ go.sum | 19 +++- internal/storage/azure/azure.go | 160 ++++++++++++++++++++++++++++++++ test/azure/docker-compose.yml | 58 ++++++++++++ test/azure/run.sh | 40 ++++++++ 9 files changed, 396 insertions(+), 50 deletions(-) create mode 100644 internal/storage/azure/azure.go create mode 100644 test/azure/docker-compose.yml create mode 100644 test/azure/run.sh diff --git a/README.md b/README.md index 495babe..ed016d8 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Backup Docker volumes locally or to any S3, WebDAV or SSH compatible storage. The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup. -It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__. +It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV, Azure Blob Storage or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__. @@ -41,6 +41,7 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc - [Backing up to MinIO \(using Docker secrets\)](#backing-up-to-minio-using-docker-secrets) - [Backing up to WebDAV](#backing-up-to-webdav) - [Backing up to SSH](#backing-up-to-ssh) + - [Backing up to Azure Blob Storage](#backing-up-to-azure-blob-storage) - [Backing up locally](#backing-up-locally) - [Backing up to AWS S3 as well as locally](#backing-up-to-aws-s3-as-well-as-locally) - [Running on a custom cron schedule](#running-on-a-custom-cron-schedule) @@ -304,6 +305,25 @@ You can populate below template according to your requirements and use it as you # SSH_IDENTITY_PASSPHRASE="pass" +# The credential's account name when using Azure Blob Storage. This has to be +# set when using Azure Blob Storage. + +# AZURE_STORAGE_ACCOUNT_NAME="account-name" + +# The credential's primary account key when using Azure Blob Storage. If this +# is not given, the command tries to fall back to using a managed identity. + +# AZURE_STORAGE_PRIMARY_ACCOUNT_KEY="" + +# The container name when using Azure Blob Storage. + +# AZURE_STORAGE_CONTAINER_NAME="container-name" + +# The service endpoint when using Azure Blob Storage. This is a template that +# can be passed the account name as shown in the default value below. + +# AZURE_STORAGE_ENDPOINT="https://{{ .AccountName }}.blob.core.windows.net/" + # In addition to storing backups remotely, you can also keep local copies. # Pass a container-local path to store your backups if needed. You also need to # mount a local folder or Docker volume into that location (`/archive` @@ -1079,6 +1099,27 @@ volumes: data: ``` +### Backing up to Azure Blob Storage + +```yml +version: '3' + +services: + # ... define other services using the `data` volume here + backup: + image: offen/docker-volume-backup:v2 + environment: + AZURE_STORAGE_CONTAINER_NAME: backup-container + AZURE_STORAGE_ACCOUNT_NAME: account-name + AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + volumes: + - data:/backup/my-app-backup:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + +volumes: + data: +``` + ### Backing up locally ```yml diff --git a/cmd/backup/config.go b/cmd/backup/config.go index 8743cf4..88ebba1 100644 --- a/cmd/backup/config.go +++ b/cmd/backup/config.go @@ -16,53 +16,58 @@ import ( // Config holds all configuration values that are expected to be set // by users. type Config struct { - AwsS3BucketName string `split_words:"true"` - AwsS3Path string `split_words:"true"` - AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` - AwsEndpointProto string `split_words:"true" default:"https"` - AwsEndpointInsecure bool `split_words:"true"` - AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` - AwsStorageClass string `split_words:"true"` - AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` - AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"` - AwsSecretAccessKey string `split_words:"true"` - AwsSecretAccessKeyFile string `split_words:"true"` - AwsIamRoleEndpoint string `split_words:"true"` - BackupSources string `split_words:"true" default:"/backup"` - BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` - BackupFilenameExpand bool `split_words:"true"` - BackupLatestSymlink string `split_words:"true"` - BackupArchive string `split_words:"true" default:"/archive"` - BackupRetentionDays int32 `split_words:"true" default:"-1"` - BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` - BackupPruningPrefix string `split_words:"true"` - BackupStopContainerLabel string `split_words:"true" default:"true"` - BackupFromSnapshot bool `split_words:"true"` - BackupExcludeRegexp RegexpDecoder `split_words:"true"` - GpgPassphrase string `split_words:"true"` - NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` - NotificationLevel string `split_words:"true" default:"error"` - EmailNotificationRecipient string `split_words:"true"` - EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` - EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` - EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` - EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` - EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` - WebdavUrl string `split_words:"true"` - WebdavUrlInsecure bool `split_words:"true"` - WebdavPath string `split_words:"true" default:"/"` - WebdavUsername string `split_words:"true"` - WebdavPassword string `split_words:"true"` - SSHHostName string `split_words:"true"` - SSHPort string `split_words:"true" default:"22"` - SSHUser string `split_words:"true"` - SSHPassword string `split_words:"true"` - SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"` - SSHIdentityPassphrase string `split_words:"true"` - SSHRemotePath string `split_words:"true"` - ExecLabel string `split_words:"true"` - ExecForwardOutput bool `split_words:"true"` - LockTimeout time.Duration `split_words:"true" default:"60m"` + AwsS3BucketName string `split_words:"true"` + AwsS3Path string `split_words:"true"` + AwsEndpoint string `split_words:"true" default:"s3.amazonaws.com"` + AwsEndpointProto string `split_words:"true" default:"https"` + AwsEndpointInsecure bool `split_words:"true"` + AwsEndpointCACert CertDecoder `envconfig:"AWS_ENDPOINT_CA_CERT"` + AwsStorageClass string `split_words:"true"` + AwsAccessKeyID string `envconfig:"AWS_ACCESS_KEY_ID"` + AwsAccessKeyIDFile string `envconfig:"AWS_ACCESS_KEY_ID_FILE"` + AwsSecretAccessKey string `split_words:"true"` + AwsSecretAccessKeyFile string `split_words:"true"` + AwsIamRoleEndpoint string `split_words:"true"` + BackupSources string `split_words:"true" default:"/backup"` + BackupFilename string `split_words:"true" default:"backup-%Y-%m-%dT%H-%M-%S.tar.gz"` + BackupFilenameExpand bool `split_words:"true"` + BackupLatestSymlink string `split_words:"true"` + BackupArchive string `split_words:"true" default:"/archive"` + BackupRetentionDays int32 `split_words:"true" default:"-1"` + BackupPruningLeeway time.Duration `split_words:"true" default:"1m"` + BackupPruningPrefix string `split_words:"true"` + BackupStopContainerLabel string `split_words:"true" default:"true"` + BackupFromSnapshot bool `split_words:"true"` + BackupExcludeRegexp RegexpDecoder `split_words:"true"` + GpgPassphrase string `split_words:"true"` + NotificationURLs []string `envconfig:"NOTIFICATION_URLS"` + NotificationLevel string `split_words:"true" default:"error"` + EmailNotificationRecipient string `split_words:"true"` + EmailNotificationSender string `split_words:"true" default:"noreply@nohost"` + EmailSMTPHost string `envconfig:"EMAIL_SMTP_HOST"` + EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"` + EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"` + EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"` + WebdavUrl string `split_words:"true"` + WebdavUrlInsecure bool `split_words:"true"` + WebdavPath string `split_words:"true" default:"/"` + WebdavUsername string `split_words:"true"` + WebdavPassword string `split_words:"true"` + SSHHostName string `split_words:"true"` + SSHPort string `split_words:"true" default:"22"` + SSHUser string `split_words:"true"` + SSHPassword string `split_words:"true"` + SSHIdentityFile string `split_words:"true" default:"/root/.ssh/id_rsa"` + SSHIdentityPassphrase string `split_words:"true"` + SSHRemotePath string `split_words:"true"` + ExecLabel string `split_words:"true"` + ExecForwardOutput bool `split_words:"true"` + LockTimeout time.Duration `split_words:"true" default:"60m"` + AzureStorageAccountName string `split_words:"true"` + AzureStoragePrimaryAccountKey string `split_words:"true"` + AzureStorageContainerName string `split_words:"true"` + AzureStoragePath string `split_words:"true"` + AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"` } func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) { diff --git a/cmd/backup/script.go b/cmd/backup/script.go index ab7447b..c4d34ac 100644 --- a/cmd/backup/script.go +++ b/cmd/backup/script.go @@ -15,6 +15,7 @@ import ( "time" "github.com/offen/docker-volume-backup/internal/storage" + "github.com/offen/docker-volume-backup/internal/storage/azure" "github.com/offen/docker-volume-backup/internal/storage/local" "github.com/offen/docker-volume-backup/internal/storage/s3" "github.com/offen/docker-volume-backup/internal/storage/ssh" @@ -76,6 +77,7 @@ func newScript() (*script, error) { "WebDAV": {}, "SSH": {}, "Local": {}, + "Azure": {}, }, }, } @@ -189,6 +191,21 @@ func newScript() (*script, error) { s.storages = append(s.storages, localBackend) } + if s.c.AzureStorageAccountName != "" { + azureConfig := azure.Config{ + ContainerName: s.c.AzureStorageContainerName, + AccountName: s.c.AzureStorageAccountName, + PrimaryAccountKey: s.c.AzureStoragePrimaryAccountKey, + Endpoint: s.c.AzureStorageEndpoint, + RemotePath: s.c.AzureStoragePath, + } + azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc) + if err != nil { + return nil, err + } + s.storages = append(s.storages, azureBackend) + } + if s.c.EmailNotificationRecipient != "" { emailURL := fmt.Sprintf( "smtp://%s:%s@%s:%d/?from=%s&to=%s", diff --git a/docs/NOTIFICATION-TEMPLATES.md b/docs/NOTIFICATION-TEMPLATES.md index 8af37af..8bd5d60 100644 --- a/docs/NOTIFICATION-TEMPLATES.md +++ b/docs/NOTIFICATION-TEMPLATES.md @@ -25,7 +25,7 @@ Here is a list of all data passed to the template: * `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`) * `Size`: size in bytes of the backup file * `Storages`: object that holds stats about each storage - * `Local`, `S3`, `WebDAV` or `SSH`: + * `Local`, `S3`, `WebDAV`, `Azure` or `SSH`: * `Total`: total number of backup files * `Pruned`: number of backup files that were deleted due to pruning rule * `PruneErrors`: number of backup files that were unable to be pruned diff --git a/go.mod b/go.mod index 96e6369..b852f76 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/offen/docker-volume-backup go 1.19 require ( + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 github.com/containrrr/shoutrrr v0.5.2 github.com/cosiner/argv v0.1.0 github.com/docker/docker v20.10.11+incompatible @@ -19,6 +21,9 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/containerd/containerd v1.6.6 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect @@ -28,6 +33,7 @@ require ( github.com/fatih/color v1.10.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/uuid v1.3.0 // indirect github.com/gorilla/mux v1.7.3 // indirect @@ -36,6 +42,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.1 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/minio/md5-simd v1.1.2 // indirect @@ -50,6 +57,7 @@ require ( github.com/onsi/gomega v1.10.3 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect + github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rs/xid v1.4.0 // indirect golang.org/x/net v0.2.0 // indirect diff --git a/go.sum b/go.sum index 50b063f..30368f7 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,17 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM= +github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -48,6 +58,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo= @@ -94,6 +105,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -174,6 +187,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs= github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -244,6 +259,8 @@ github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI= +github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -296,8 +313,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9HY7giGM+kYcnQ71m14JnGdQabMPmyt++8= github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= diff --git a/internal/storage/azure/azure.go b/internal/storage/azure/azure.go new file mode 100644 index 0000000..c6c1d80 --- /dev/null +++ b/internal/storage/azure/azure.go @@ -0,0 +1,160 @@ +// Copyright 2022 - Offen Authors +// SPDX-License-Identifier: MPL-2.0 + +package azure + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "text/template" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/offen/docker-volume-backup/internal/storage" + "github.com/offen/docker-volume-backup/internal/utilities" +) + +type azureBlobStorage struct { + *storage.StorageBackend + client *azblob.Client + containerName string +} + +// Config contains values that define the configuration of an Azure Blob Storage. +type Config struct { + AccountName string + ContainerName string + PrimaryAccountKey string + Endpoint string + RemotePath string +} + +// NewStorageBackend creates and initializes a new Azure Blob Storage backend. +func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { + endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error parsing endpoint template: %w", err) + } + var ep bytes.Buffer + if err := endpointTemplate.Execute(&ep, opts); err != nil { + return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err) + } + normalizedEndpoint := fmt.Sprintf("%s/", strings.TrimSuffix(ep.String(), "/")) + + var client *azblob.Client + if opts.PrimaryAccountKey != "" { + cred, err := azblob.NewSharedKeyCredential(opts.AccountName, opts.PrimaryAccountKey) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating shared key Azure credential: %w", err) + } + + client, err = azblob.NewClientWithSharedKeyCredential(normalizedEndpoint, cred, nil) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) + } + } else { + cred, err := azidentity.NewManagedIdentityCredential(nil) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating managed identity credential: %w", err) + } + client, err = azblob.NewClient(normalizedEndpoint, cred, nil) + if err != nil { + return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err) + } + } + + storage := azureBlobStorage{ + client: client, + containerName: opts.ContainerName, + StorageBackend: &storage.StorageBackend{ + DestinationPath: opts.RemotePath, + Log: logFunc, + }, + } + return &storage, nil +} + +// Name returns the name of the storage backend +func (b *azureBlobStorage) Name() string { + return "Azure" +} + +// Copy copies the given file to the storage backend. +func (b *azureBlobStorage) Copy(file string) error { + fileReader, err := os.Open(file) + if err != nil { + return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err) + } + _, err = b.client.UploadStream( + context.Background(), + b.containerName, + filepath.Join(b.DestinationPath, filepath.Base(file)), + fileReader, + nil, + ) + if err != nil { + return fmt.Errorf("(*azureBlobStorage).Copy: error uploading file %s: %w", file, err) + } + return nil +} + +// Prune rotates away backups according to the configuration and provided +// deadline for the Azure Blob storage backend. +func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { + lookupPrefix := filepath.Join(b.DestinationPath, pruningPrefix) + pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{ + Prefix: &lookupPrefix, + }) + var matches []string + var totalCount uint + for pager.More() { + resp, err := pager.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("(*azureBlobStorage).Prune: error paging over blobs: %w", err) + } + for _, v := range resp.Segment.BlobItems { + totalCount++ + if v.Properties.LastModified.Before(deadline) { + matches = append(matches, *v.Name) + } + } + } + + stats := storage.PruneStats{ + Total: totalCount, + Pruned: uint(len(matches)), + } + + if err := b.DoPrune(b.Name(), len(matches), int(totalCount), "Azure Blob Storage backup(s)", func() error { + wg := sync.WaitGroup{} + wg.Add(len(matches)) + var errors []error + + for _, match := range matches { + name := match + go func() { + _, err := b.client.DeleteBlob(context.Background(), b.containerName, name, nil) + if err != nil { + errors = append(errors, err) + } + wg.Done() + }() + } + wg.Wait() + if len(errors) != 0 { + return utilities.Join(errors...) + } + return nil + }); err != nil { + return &stats, err + } + + return &stats, nil +} diff --git a/test/azure/docker-compose.yml b/test/azure/docker-compose.yml new file mode 100644 index 0000000..34eab22 --- /dev/null +++ b/test/azure/docker-compose.yml @@ -0,0 +1,58 @@ +version: '3' + +services: + storage: + image: mcr.microsoft.com/azure-storage/azurite + volumes: + - azurite_backup_data:/data + command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data + healthcheck: + test: nc 127.0.0.1 10000 -z + interval: 1s + retries: 30 + + az_cli: + image: mcr.microsoft.com/azure-cli + volumes: + - ./local:/dump + command: + - /bin/sh + - -c + - | + az storage container create --name test-container + depends_on: + storage: + condition: service_healthy + environment: + AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1; + + backup: + image: offen/docker-volume-backup:${TEST_VERSION:-canary} + hostname: hostnametoken + restart: always + environment: + AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1 + AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + AZURE_STORAGE_CONTAINER_NAME: test-container + AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/ + AZURE_STORAGE_PATH: 'path/to/backup' + BACKUP_FILENAME: test.tar.gz + BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} + BACKUP_PRUNING_LEEWAY: 5s + BACKUP_PRUNING_PREFIX: test + volumes: + - app_data:/backup/app_data:ro + - /var/run/docker.sock:/var/run/docker.sock + + offen: + image: offen/offen:latest + labels: + - docker-volume-backup.stop-during-backup=true + volumes: + - app_data:/var/opt/offen + +volumes: + azurite_backup_data: + name: azurite_backup_data + app_data: diff --git a/test/azure/run.sh b/test/azure/run.sh new file mode 100644 index 0000000..0077706 --- /dev/null +++ b/test/azure/run.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")" +. ../util.sh +current_test=$(basename $(pwd)) + +docker-compose up -d +sleep 5 + +# A symlink for a known file in the volume is created so the test can check +# whether symlinks are preserved on backup. +docker-compose exec backup backup + +sleep 5 + +expect_running_containers "3" + +docker-compose run --rm az_cli \ + az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz +tar -xvf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db + +pass "Found relevant files in untared remote backups." + +# The second part of this test checks if backups get deleted when the retention +# is set to 0 days (which it should not as it would mean all backups get deleted) +# TODO: find out if we can test actual deletion without having to wait for a day +BACKUP_RETENTION_DAYS="0" docker-compose up -d +sleep 5 + +docker-compose exec backup backup + +docker-compose run --rm az_cli \ + az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz +test -f ./local/test.tar.gz + +pass "Remote backups have not been deleted." + +docker-compose down --volumes