mirror of
https://github.com/offen/docker-volume-backup.git
synced 2024-11-22 05:10:28 +01:00
Add support for Azure Blob Storage (#171)
* Scaffold Azure storage backend that does nothing yet * Implement copy for Azure Blob Storage * Set up automated testing for Azure Storage * Implement pruning for Azure blob storage * Add documentation for Azure Blob Storage * Add support for remote path * Add azure to notifications doc * Tidy go.mod file * Allow use of managed identity credential * Use volume in tests * Auto append trailing slash to endpoint if needed, clarify docs, tidy mod file
This commit is contained in:
parent
aadbaa741d
commit
5c04e11f10
43
README.md
43
README.md
@ -7,7 +7,7 @@
|
|||||||
Backup Docker volumes locally or to any S3, WebDAV or SSH compatible storage.
|
Backup Docker volumes locally or to any S3, WebDAV or SSH compatible storage.
|
||||||
|
|
||||||
The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup.
|
The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) sidecar container to an existing Docker setup.
|
||||||
It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__.
|
It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV, Azure Blob Storage or SSH compatible storage (or any combination) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for failed backup runs__.
|
||||||
|
|
||||||
<!-- MarkdownTOC -->
|
<!-- MarkdownTOC -->
|
||||||
|
|
||||||
@ -41,6 +41,7 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
|
|||||||
- [Backing up to MinIO \(using Docker secrets\)](#backing-up-to-minio-using-docker-secrets)
|
- [Backing up to MinIO \(using Docker secrets\)](#backing-up-to-minio-using-docker-secrets)
|
||||||
- [Backing up to WebDAV](#backing-up-to-webdav)
|
- [Backing up to WebDAV](#backing-up-to-webdav)
|
||||||
- [Backing up to SSH](#backing-up-to-ssh)
|
- [Backing up to SSH](#backing-up-to-ssh)
|
||||||
|
- [Backing up to Azure Blob Storage](#backing-up-to-azure-blob-storage)
|
||||||
- [Backing up locally](#backing-up-locally)
|
- [Backing up locally](#backing-up-locally)
|
||||||
- [Backing up to AWS S3 as well as locally](#backing-up-to-aws-s3-as-well-as-locally)
|
- [Backing up to AWS S3 as well as locally](#backing-up-to-aws-s3-as-well-as-locally)
|
||||||
- [Running on a custom cron schedule](#running-on-a-custom-cron-schedule)
|
- [Running on a custom cron schedule](#running-on-a-custom-cron-schedule)
|
||||||
@ -304,6 +305,25 @@ You can populate below template according to your requirements and use it as you
|
|||||||
|
|
||||||
# SSH_IDENTITY_PASSPHRASE="pass"
|
# SSH_IDENTITY_PASSPHRASE="pass"
|
||||||
|
|
||||||
|
# The credential's account name when using Azure Blob Storage. This has to be
|
||||||
|
# set when using Azure Blob Storage.
|
||||||
|
|
||||||
|
# AZURE_STORAGE_ACCOUNT_NAME="account-name"
|
||||||
|
|
||||||
|
# The credential's primary account key when using Azure Blob Storage. If this
|
||||||
|
# is not given, the command tries to fall back to using a managed identity.
|
||||||
|
|
||||||
|
# AZURE_STORAGE_PRIMARY_ACCOUNT_KEY="<xxx>"
|
||||||
|
|
||||||
|
# The container name when using Azure Blob Storage.
|
||||||
|
|
||||||
|
# AZURE_STORAGE_CONTAINER_NAME="container-name"
|
||||||
|
|
||||||
|
# The service endpoint when using Azure Blob Storage. This is a template that
|
||||||
|
# can be passed the account name as shown in the default value below.
|
||||||
|
|
||||||
|
# AZURE_STORAGE_ENDPOINT="https://{{ .AccountName }}.blob.core.windows.net/"
|
||||||
|
|
||||||
# In addition to storing backups remotely, you can also keep local copies.
|
# In addition to storing backups remotely, you can also keep local copies.
|
||||||
# Pass a container-local path to store your backups if needed. You also need to
|
# Pass a container-local path to store your backups if needed. You also need to
|
||||||
# mount a local folder or Docker volume into that location (`/archive`
|
# mount a local folder or Docker volume into that location (`/archive`
|
||||||
@ -1079,6 +1099,27 @@ volumes:
|
|||||||
data:
|
data:
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Backing up to Azure Blob Storage
|
||||||
|
|
||||||
|
```yml
|
||||||
|
version: '3'
|
||||||
|
|
||||||
|
services:
|
||||||
|
# ... define other services using the `data` volume here
|
||||||
|
backup:
|
||||||
|
image: offen/docker-volume-backup:v2
|
||||||
|
environment:
|
||||||
|
AZURE_STORAGE_CONTAINER_NAME: backup-container
|
||||||
|
AZURE_STORAGE_ACCOUNT_NAME: account-name
|
||||||
|
AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
|
||||||
|
volumes:
|
||||||
|
- data:/backup/my-app-backup:ro
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
data:
|
||||||
|
```
|
||||||
|
|
||||||
### Backing up locally
|
### Backing up locally
|
||||||
|
|
||||||
```yml
|
```yml
|
||||||
|
@ -63,6 +63,11 @@ type Config struct {
|
|||||||
ExecLabel string `split_words:"true"`
|
ExecLabel string `split_words:"true"`
|
||||||
ExecForwardOutput bool `split_words:"true"`
|
ExecForwardOutput bool `split_words:"true"`
|
||||||
LockTimeout time.Duration `split_words:"true" default:"60m"`
|
LockTimeout time.Duration `split_words:"true" default:"60m"`
|
||||||
|
AzureStorageAccountName string `split_words:"true"`
|
||||||
|
AzureStoragePrimaryAccountKey string `split_words:"true"`
|
||||||
|
AzureStorageContainerName string `split_words:"true"`
|
||||||
|
AzureStoragePath string `split_words:"true"`
|
||||||
|
AzureStorageEndpoint string `split_words:"true" default:"https://{{ .AccountName }}.blob.core.windows.net/"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
|
func (c *Config) resolveSecret(envVar string, secretPath string) (string, error) {
|
||||||
|
@ -15,6 +15,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/offen/docker-volume-backup/internal/storage"
|
"github.com/offen/docker-volume-backup/internal/storage"
|
||||||
|
"github.com/offen/docker-volume-backup/internal/storage/azure"
|
||||||
"github.com/offen/docker-volume-backup/internal/storage/local"
|
"github.com/offen/docker-volume-backup/internal/storage/local"
|
||||||
"github.com/offen/docker-volume-backup/internal/storage/s3"
|
"github.com/offen/docker-volume-backup/internal/storage/s3"
|
||||||
"github.com/offen/docker-volume-backup/internal/storage/ssh"
|
"github.com/offen/docker-volume-backup/internal/storage/ssh"
|
||||||
@ -76,6 +77,7 @@ func newScript() (*script, error) {
|
|||||||
"WebDAV": {},
|
"WebDAV": {},
|
||||||
"SSH": {},
|
"SSH": {},
|
||||||
"Local": {},
|
"Local": {},
|
||||||
|
"Azure": {},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -189,6 +191,21 @@ func newScript() (*script, error) {
|
|||||||
s.storages = append(s.storages, localBackend)
|
s.storages = append(s.storages, localBackend)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if s.c.AzureStorageAccountName != "" {
|
||||||
|
azureConfig := azure.Config{
|
||||||
|
ContainerName: s.c.AzureStorageContainerName,
|
||||||
|
AccountName: s.c.AzureStorageAccountName,
|
||||||
|
PrimaryAccountKey: s.c.AzureStoragePrimaryAccountKey,
|
||||||
|
Endpoint: s.c.AzureStorageEndpoint,
|
||||||
|
RemotePath: s.c.AzureStoragePath,
|
||||||
|
}
|
||||||
|
azureBackend, err := azure.NewStorageBackend(azureConfig, logFunc)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
s.storages = append(s.storages, azureBackend)
|
||||||
|
}
|
||||||
|
|
||||||
if s.c.EmailNotificationRecipient != "" {
|
if s.c.EmailNotificationRecipient != "" {
|
||||||
emailURL := fmt.Sprintf(
|
emailURL := fmt.Sprintf(
|
||||||
"smtp://%s:%s@%s:%d/?from=%s&to=%s",
|
"smtp://%s:%s@%s:%d/?from=%s&to=%s",
|
||||||
|
@ -25,7 +25,7 @@ Here is a list of all data passed to the template:
|
|||||||
* `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
|
* `FullPath`: full path of the backup file (e.g. `/archive/backup-2022-02-11T01-00-00.tar.gz`)
|
||||||
* `Size`: size in bytes of the backup file
|
* `Size`: size in bytes of the backup file
|
||||||
* `Storages`: object that holds stats about each storage
|
* `Storages`: object that holds stats about each storage
|
||||||
* `Local`, `S3`, `WebDAV` or `SSH`:
|
* `Local`, `S3`, `WebDAV`, `Azure` or `SSH`:
|
||||||
* `Total`: total number of backup files
|
* `Total`: total number of backup files
|
||||||
* `Pruned`: number of backup files that were deleted due to pruning rule
|
* `Pruned`: number of backup files that were deleted due to pruning rule
|
||||||
* `PruneErrors`: number of backup files that were unable to be pruned
|
* `PruneErrors`: number of backup files that were unable to be pruned
|
||||||
|
8
go.mod
8
go.mod
@ -3,6 +3,8 @@ module github.com/offen/docker-volume-backup
|
|||||||
go 1.19
|
go 1.19
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1
|
||||||
github.com/containrrr/shoutrrr v0.5.2
|
github.com/containrrr/shoutrrr v0.5.2
|
||||||
github.com/cosiner/argv v0.1.0
|
github.com/cosiner/argv v0.1.0
|
||||||
github.com/docker/docker v20.10.11+incompatible
|
github.com/docker/docker v20.10.11+incompatible
|
||||||
@ -19,6 +21,9 @@ require (
|
|||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 // indirect
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
|
||||||
|
github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 // indirect
|
||||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||||
github.com/containerd/containerd v1.6.6 // indirect
|
github.com/containerd/containerd v1.6.6 // indirect
|
||||||
github.com/docker/distribution v2.7.1+incompatible // indirect
|
github.com/docker/distribution v2.7.1+incompatible // indirect
|
||||||
@ -28,6 +33,7 @@ require (
|
|||||||
github.com/fatih/color v1.10.0 // indirect
|
github.com/fatih/color v1.10.0 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
github.com/fsnotify/fsnotify v1.4.9 // indirect
|
||||||
github.com/gogo/protobuf v1.3.2 // indirect
|
github.com/gogo/protobuf v1.3.2 // indirect
|
||||||
|
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
|
||||||
github.com/golang/protobuf v1.5.2 // indirect
|
github.com/golang/protobuf v1.5.2 // indirect
|
||||||
github.com/google/uuid v1.3.0 // indirect
|
github.com/google/uuid v1.3.0 // indirect
|
||||||
github.com/gorilla/mux v1.7.3 // indirect
|
github.com/gorilla/mux v1.7.3 // indirect
|
||||||
@ -36,6 +42,7 @@ require (
|
|||||||
github.com/klauspost/cpuid/v2 v2.2.1 // indirect
|
github.com/klauspost/cpuid/v2 v2.2.1 // indirect
|
||||||
github.com/kr/fs v0.1.0 // indirect
|
github.com/kr/fs v0.1.0 // indirect
|
||||||
github.com/kr/text v0.2.0 // indirect
|
github.com/kr/text v0.2.0 // indirect
|
||||||
|
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||||
github.com/mattn/go-colorable v0.1.8 // indirect
|
github.com/mattn/go-colorable v0.1.8 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.12 // indirect
|
github.com/mattn/go-isatty v0.0.12 // indirect
|
||||||
github.com/minio/md5-simd v1.1.2 // indirect
|
github.com/minio/md5-simd v1.1.2 // indirect
|
||||||
@ -50,6 +57,7 @@ require (
|
|||||||
github.com/onsi/gomega v1.10.3 // indirect
|
github.com/onsi/gomega v1.10.3 // indirect
|
||||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||||
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
|
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
|
||||||
|
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
github.com/rs/xid v1.4.0 // indirect
|
github.com/rs/xid v1.4.0 // indirect
|
||||||
golang.org/x/net v0.2.0 // indirect
|
golang.org/x/net v0.2.0 // indirect
|
||||||
|
19
go.sum
19
go.sum
@ -1,7 +1,17 @@
|
|||||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4 h1:pqrAR74b6EoR4kcxF7L7Wg2B8Jgil9UUZtMvxhEFqWo=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0 h1:t/W5MYAuQy81cvM8VUNfRLzhtKpXhVUAN7Cd7KVbTyc=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.0/go.mod h1:NBanQUfSWiWn3QEpWDTCU0IjBECKOYvl2R8xdRtMtiM=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQiecy1SI+Eog7xVkPiqIbg=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1 h1:YvQv9Mz6T8oR5ypQOL6erY0Z5t71ak1uHV4QFokCOZk=
|
||||||
|
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.6.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
|
||||||
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
|
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
|
||||||
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
||||||
|
github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0 h1:VgSJlZH5u0k2qxSpqyghcFQKmvYckj46uymKK5XzkBM=
|
||||||
|
github.com/AzureAD/microsoft-authentication-library-for-go v0.7.0/go.mod h1:BDJ5qMFKx9DugEg3+uQSDCdbYPr5s9vBTrL9P8TpqOU=
|
||||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
|
github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
|
||||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||||
@ -48,6 +58,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
|||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||||
|
github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c=
|
||||||
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
|
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
|
||||||
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||||
github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo=
|
github.com/docker/docker v20.10.11+incompatible h1:OqzI/g/W54LczvhnccGqniFoQghHx3pklbLuhfXpqGo=
|
||||||
@ -94,6 +105,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
|
|||||||
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
|
||||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||||
|
github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
|
||||||
|
github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||||
@ -174,6 +187,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|||||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
|
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||||
|
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||||
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs=
|
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs=
|
||||||
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4=
|
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4=
|
||||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||||
@ -244,6 +259,8 @@ github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI=
|
|||||||
github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
|
github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
|
||||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||||
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
|
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
|
||||||
|
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
|
||||||
|
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
|
||||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
@ -296,8 +313,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
|
|||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
|
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
|
||||||
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9HY7giGM+kYcnQ71m14JnGdQabMPmyt++8=
|
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9HY7giGM+kYcnQ71m14JnGdQabMPmyt++8=
|
||||||
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
|
github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
|
||||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||||
|
160
internal/storage/azure/azure.go
Normal file
160
internal/storage/azure/azure.go
Normal file
@ -0,0 +1,160 @@
|
|||||||
|
// Copyright 2022 - Offen Authors <hioffen@posteo.de>
|
||||||
|
// SPDX-License-Identifier: MPL-2.0
|
||||||
|
|
||||||
|
package azure
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"text/template"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||||
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||||
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||||
|
"github.com/offen/docker-volume-backup/internal/storage"
|
||||||
|
"github.com/offen/docker-volume-backup/internal/utilities"
|
||||||
|
)
|
||||||
|
|
||||||
|
type azureBlobStorage struct {
|
||||||
|
*storage.StorageBackend
|
||||||
|
client *azblob.Client
|
||||||
|
containerName string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config contains values that define the configuration of an Azure Blob Storage.
|
||||||
|
type Config struct {
|
||||||
|
AccountName string
|
||||||
|
ContainerName string
|
||||||
|
PrimaryAccountKey string
|
||||||
|
Endpoint string
|
||||||
|
RemotePath string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStorageBackend creates and initializes a new Azure Blob Storage backend.
|
||||||
|
func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
|
||||||
|
endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("NewStorageBackend: error parsing endpoint template: %w", err)
|
||||||
|
}
|
||||||
|
var ep bytes.Buffer
|
||||||
|
if err := endpointTemplate.Execute(&ep, opts); err != nil {
|
||||||
|
return nil, fmt.Errorf("NewStorageBackend: error executing endpoint template: %w", err)
|
||||||
|
}
|
||||||
|
normalizedEndpoint := fmt.Sprintf("%s/", strings.TrimSuffix(ep.String(), "/"))
|
||||||
|
|
||||||
|
var client *azblob.Client
|
||||||
|
if opts.PrimaryAccountKey != "" {
|
||||||
|
cred, err := azblob.NewSharedKeyCredential(opts.AccountName, opts.PrimaryAccountKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("NewStorageBackend: error creating shared key Azure credential: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err = azblob.NewClientWithSharedKeyCredential(normalizedEndpoint, cred, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
cred, err := azidentity.NewManagedIdentityCredential(nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("NewStorageBackend: error creating managed identity credential: %w", err)
|
||||||
|
}
|
||||||
|
client, err = azblob.NewClient(normalizedEndpoint, cred, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("NewStorageBackend: error creating Azure client: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
storage := azureBlobStorage{
|
||||||
|
client: client,
|
||||||
|
containerName: opts.ContainerName,
|
||||||
|
StorageBackend: &storage.StorageBackend{
|
||||||
|
DestinationPath: opts.RemotePath,
|
||||||
|
Log: logFunc,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return &storage, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the name of the storage backend
|
||||||
|
func (b *azureBlobStorage) Name() string {
|
||||||
|
return "Azure"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copy copies the given file to the storage backend.
|
||||||
|
func (b *azureBlobStorage) Copy(file string) error {
|
||||||
|
fileReader, err := os.Open(file)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("(*azureBlobStorage).Copy: error opening file %s: %w", file, err)
|
||||||
|
}
|
||||||
|
_, err = b.client.UploadStream(
|
||||||
|
context.Background(),
|
||||||
|
b.containerName,
|
||||||
|
filepath.Join(b.DestinationPath, filepath.Base(file)),
|
||||||
|
fileReader,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("(*azureBlobStorage).Copy: error uploading file %s: %w", file, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prune rotates away backups according to the configuration and provided
|
||||||
|
// deadline for the Azure Blob storage backend.
|
||||||
|
func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) {
|
||||||
|
lookupPrefix := filepath.Join(b.DestinationPath, pruningPrefix)
|
||||||
|
pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{
|
||||||
|
Prefix: &lookupPrefix,
|
||||||
|
})
|
||||||
|
var matches []string
|
||||||
|
var totalCount uint
|
||||||
|
for pager.More() {
|
||||||
|
resp, err := pager.NextPage(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("(*azureBlobStorage).Prune: error paging over blobs: %w", err)
|
||||||
|
}
|
||||||
|
for _, v := range resp.Segment.BlobItems {
|
||||||
|
totalCount++
|
||||||
|
if v.Properties.LastModified.Before(deadline) {
|
||||||
|
matches = append(matches, *v.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := storage.PruneStats{
|
||||||
|
Total: totalCount,
|
||||||
|
Pruned: uint(len(matches)),
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := b.DoPrune(b.Name(), len(matches), int(totalCount), "Azure Blob Storage backup(s)", func() error {
|
||||||
|
wg := sync.WaitGroup{}
|
||||||
|
wg.Add(len(matches))
|
||||||
|
var errors []error
|
||||||
|
|
||||||
|
for _, match := range matches {
|
||||||
|
name := match
|
||||||
|
go func() {
|
||||||
|
_, err := b.client.DeleteBlob(context.Background(), b.containerName, name, nil)
|
||||||
|
if err != nil {
|
||||||
|
errors = append(errors, err)
|
||||||
|
}
|
||||||
|
wg.Done()
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
if len(errors) != 0 {
|
||||||
|
return utilities.Join(errors...)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return &stats, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &stats, nil
|
||||||
|
}
|
58
test/azure/docker-compose.yml
Normal file
58
test/azure/docker-compose.yml
Normal file
@ -0,0 +1,58 @@
|
|||||||
|
version: '3'
|
||||||
|
|
||||||
|
services:
|
||||||
|
storage:
|
||||||
|
image: mcr.microsoft.com/azure-storage/azurite
|
||||||
|
volumes:
|
||||||
|
- azurite_backup_data:/data
|
||||||
|
command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data
|
||||||
|
healthcheck:
|
||||||
|
test: nc 127.0.0.1 10000 -z
|
||||||
|
interval: 1s
|
||||||
|
retries: 30
|
||||||
|
|
||||||
|
az_cli:
|
||||||
|
image: mcr.microsoft.com/azure-cli
|
||||||
|
volumes:
|
||||||
|
- ./local:/dump
|
||||||
|
command:
|
||||||
|
- /bin/sh
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
az storage container create --name test-container
|
||||||
|
depends_on:
|
||||||
|
storage:
|
||||||
|
condition: service_healthy
|
||||||
|
environment:
|
||||||
|
AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1;
|
||||||
|
|
||||||
|
backup:
|
||||||
|
image: offen/docker-volume-backup:${TEST_VERSION:-canary}
|
||||||
|
hostname: hostnametoken
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1
|
||||||
|
AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
|
||||||
|
AZURE_STORAGE_CONTAINER_NAME: test-container
|
||||||
|
AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/
|
||||||
|
AZURE_STORAGE_PATH: 'path/to/backup'
|
||||||
|
BACKUP_FILENAME: test.tar.gz
|
||||||
|
BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
|
||||||
|
BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
|
||||||
|
BACKUP_PRUNING_LEEWAY: 5s
|
||||||
|
BACKUP_PRUNING_PREFIX: test
|
||||||
|
volumes:
|
||||||
|
- app_data:/backup/app_data:ro
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
|
||||||
|
offen:
|
||||||
|
image: offen/offen:latest
|
||||||
|
labels:
|
||||||
|
- docker-volume-backup.stop-during-backup=true
|
||||||
|
volumes:
|
||||||
|
- app_data:/var/opt/offen
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
azurite_backup_data:
|
||||||
|
name: azurite_backup_data
|
||||||
|
app_data:
|
40
test/azure/run.sh
Normal file
40
test/azure/run.sh
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
. ../util.sh
|
||||||
|
current_test=$(basename $(pwd))
|
||||||
|
|
||||||
|
docker-compose up -d
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
# A symlink for a known file in the volume is created so the test can check
|
||||||
|
# whether symlinks are preserved on backup.
|
||||||
|
docker-compose exec backup backup
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
expect_running_containers "3"
|
||||||
|
|
||||||
|
docker-compose run --rm az_cli \
|
||||||
|
az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz
|
||||||
|
tar -xvf ./local/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db
|
||||||
|
|
||||||
|
pass "Found relevant files in untared remote backups."
|
||||||
|
|
||||||
|
# The second part of this test checks if backups get deleted when the retention
|
||||||
|
# is set to 0 days (which it should not as it would mean all backups get deleted)
|
||||||
|
# TODO: find out if we can test actual deletion without having to wait for a day
|
||||||
|
BACKUP_RETENTION_DAYS="0" docker-compose up -d
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
docker-compose exec backup backup
|
||||||
|
|
||||||
|
docker-compose run --rm az_cli \
|
||||||
|
az storage blob download -f /dump/test.tar.gz -c test-container -n path/to/backup/test.tar.gz
|
||||||
|
test -f ./local/test.tar.gz
|
||||||
|
|
||||||
|
pass "Remote backups have not been deleted."
|
||||||
|
|
||||||
|
docker-compose down --volumes
|
Loading…
Reference in New Issue
Block a user