Allow use of a custom ca cert when working against S3 storages (#170)

README.md

@@ -37,6 +37,7 @@ It handles __recurring or one-off backups of Docker volumes__ to a __local direc
   - [Backing up to AWS S3](#backing-up-to-aws-s3)
   - [Backing up to Filebase](#backing-up-to-filebase)
   - [Backing up to MinIO](#backing-up-to-minio)
+  - [Backing up to MinIO \(using Docker secrets\)](#backing-up-to-minio-using-docker-secrets)
   - [Backing up to WebDAV](#backing-up-to-webdav)
   - [Backing up to SSH](#backing-up-to-ssh)
   - [Backing up locally](#backing-up-locally)
@@ -231,6 +232,13 @@ You can populate below template according to your requirements and use it as you
 
 # AWS_ENDPOINT_INSECURE="true"
 
+# If you wish to use self signed certificates your S3 server, you can pass
+# the location of a PEM encoded CA certificate and it will be used for
+# validating your certificates.
+# Alternatively, pass a PEM encoded string containing the certificate.
+
+# AWS_ENDPOINT_CA_CERT="/path/to/cert.pem"
+
 # Setting this variable will change the S3 storage class header.
 # Defaults to "STANDARD", you can set this value according to your needs.
 

cmd/backup/config.go

@@ -4,7 +4,10 @@
 package main
 
 import (
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"regexp"
 	"time"
@@ -18,6 +21,7 @@ type Config struct {
 	AwsEndpoint                string        `split_words:"true" default:"s3.amazonaws.com"`
 	AwsEndpointProto           string        `split_words:"true" default:"https"`
 	AwsEndpointInsecure        bool          `split_words:"true"`
+	AwsEndpointCACert          CertDecoder   `envconfig:"AWS_ENDPOINT_CA_CERT"`
 	AwsStorageClass            string        `split_words:"true"`
 	AwsAccessKeyID             string        `envconfig:"AWS_ACCESS_KEY_ID"`
 	AwsAccessKeyIDFile         string        `envconfig:"AWS_ACCESS_KEY_ID_FILE"`
@@ -72,6 +76,27 @@ func (c *Config) resolveSecret(envVar string, secretPath string) (string, error)
 	return string(data), nil
 }
 
+type CertDecoder struct {
+	Cert *x509.Certificate
+}
+
+func (c *CertDecoder) Decode(v string) error {
+	if v == "" {
+		return nil
+	}
+	content, err := ioutil.ReadFile(v)
+	if err != nil {
+		content = []byte(v)
+	}
+	block, _ := pem.Decode(content)
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return fmt.Errorf("config: error parsing certificate: %w", err)
+	}
+	*c = CertDecoder{Cert: cert}
+	return nil
+}
+
 type RegexpDecoder struct {
 	Re *regexp.Regexp
 }

cmd/backup/script.go

@@ -139,6 +139,7 @@ func newScript() (*script, error) {
 			RemotePath:       s.c.AwsS3Path,
 			BucketName:       s.c.AwsS3BucketName,
 			StorageClass:     s.c.AwsStorageClass,
+			CACert:           s.c.AwsEndpointCACert.Cert,
 		}
 		if s3Backend, err := s3.NewStorageBackend(s3Config, logFunc); err != nil {
 			return nil, err

internal/storage/s3/s3.go

@@ -5,6 +5,7 @@ package s3
 
 import (
 	"context"
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"path"
@@ -35,11 +36,11 @@ type Config struct {
 	RemotePath       string
 	BucketName       string
 	StorageClass     string
+	CACert           *x509.Certificate
 }
 
 // NewStorageBackend creates and initializes a new S3/Minio storage backend.
 func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) {
-
 	var creds *credentials.Credentials
 	if opts.AccessKeyID != "" && opts.SecretAccessKey != "" {
 		creds = credentials.NewStaticV4(
@@ -58,18 +59,23 @@ func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error
 		Secure: opts.EndpointProto == "https",
 	}
 
-	if opts.EndpointInsecure {
-		if !options.Secure {
-			return nil, errors.New("NewStorageBackend: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
-		}
-
 	transport, err := minio.DefaultTransport(true)
 	if err != nil {
 		return nil, fmt.Errorf("NewStorageBackend: failed to create default minio transport: %w", err)
 	}
-		transport.TLSClientConfig.InsecureSkipVerify = true
-		options.Transport = transport
+
+	if opts.EndpointInsecure {
+		if !options.Secure {
+			return nil, errors.New("NewStorageBackend: AWS_ENDPOINT_INSECURE = true is only meaningful for https")
 		}
+		transport.TLSClientConfig.InsecureSkipVerify = true
+	} else if opts.CACert != nil {
+		if transport.TLSClientConfig.RootCAs == nil {
+			transport.TLSClientConfig.RootCAs = x509.NewCertPool()
+		}
+		transport.TLSClientConfig.RootCAs.AddCert(opts.CACert)
+	}
+	options.Transport = transport
 
 	mc, err := minio.New(opts.Endpoint, &options)
 	if err != nil {

test/certs/docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3'
+
+services:
+  minio:
+    hostname: minio.local
+    image: minio/minio:RELEASE.2020-08-04T23-10-51Z
+    environment:
+      MINIO_ROOT_USER: test
+      MINIO_ROOT_PASSWORD: test
+      MINIO_ACCESS_KEY: test
+      MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ
+    entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server --certs-dir "/certs" --address ":443" /data'
+    volumes:
+      - minio_backup_data:/data
+      - ./minio.crt:/certs/public.crt
+      - ./minio.key:/certs/private.key
+
+  backup:
+    image: offen/docker-volume-backup:${TEST_VERSION:-canary}
+    depends_on:
+      - minio
+    restart: always
+    environment:
+      BACKUP_FILENAME: test.tar.gz
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ
+      AWS_ENDPOINT: minio.local:443
+      AWS_ENDPOINT_CA_CERT: /root/minio-rootCA.crt
+      AWS_S3_BUCKET_NAME: backup
+      BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ?
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7}
+      BACKUP_PRUNING_LEEWAY: 5s
+    volumes:
+      - app_data:/backup/app_data:ro
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./rootCA.crt:/root/minio-rootCA.crt
+
+  offen:
+    image: offen/offen:latest
+    labels:
+      - docker-volume-backup.stop-during-backup=true
+    volumes:
+      - app_data:/var/opt/offen
+
+volumes:
+  minio_backup_data:
+    name: minio_backup_data
+  app_data:

test/certs/run.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")"
+. ../util.sh
+current_test=$(basename $(pwd))
+
+openssl genrsa -des3 -passout pass:test -out rootCA.key 4096
+openssl req -passin pass:test \
+  -subj "/C=DE/ST=BE/O=IntegrationTest, Inc." \
+  -x509 -new -key rootCA.key -sha256 -days 1 -out rootCA.crt
+
+openssl genrsa -out minio.key 4096
+openssl req -new -sha256 -key minio.key \
+  -subj "/C=DE/ST=BE/O=IntegrationTest, Inc./CN=minio" \
+  -out minio.csr
+
+openssl x509 -req -passin pass:test \
+  -in minio.csr \
+  -CA rootCA.crt -CAkey rootCA.key -CAcreateserial \
+  -extfile san.cnf \
+  -out minio.crt -days 1 -sha256
+
+openssl x509 -in minio.crt -noout -text
+
+docker-compose up -d
+sleep 5
+
+docker-compose exec backup backup
+
+sleep 5
+
+expect_running_containers "3"
+
+docker run --rm -it \
+  -v minio_backup_data:/minio_data \
+  alpine \
+  ash -c 'tar -xvf /minio_data/backup/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
+
+pass "Found relevant files in untared remote backups."
+
+docker-compose down --volumes

test/certs/san.cnf

@@ -0,0 +1 @@
+subjectAltName = DNS:minio.local