mirror of
https://github.com/offen/docker-volume-backup.git
synced 2024-11-10 00:30:29 +01:00
Do not skip directories when creating tar archive (#72)
* Update targz library to include potential ownership fix * Move archive logic to main repo * Remove assertions for debugging * Use relative path in assertion * Strip local part from archive location * Log when extracting in tests * Fix trimming of prfix * Add license info to archive.go file * Undo change in test assertion * Add test checking for preserved file ownership * use same postgres version in tests * Wrap errors when archiving, handle deletion at script layer
This commit is contained in:
parent
58b42b9036
commit
3ded77448c
142
cmd/backup/archive.go
Normal file
142
cmd/backup/archive.go
Normal file
@ -0,0 +1,142 @@
|
||||
// Copyright 2022 - Offen Authors <hioffen@posteo.de>
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
// Portions of this file are taken from package `targz`, Copyright (c) 2014 Fredrik Wallgren
|
||||
// Licensed under the MIT License: https://github.com/walle/targz/blob/57fe4206da5abf7dd3901b4af3891ec2f08c7b08/LICENSE
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func createArchive(inputFilePath, outputFilePath string) error {
|
||||
inputFilePath = stripTrailingSlashes(inputFilePath)
|
||||
inputFilePath, outputFilePath, err := makeAbsolute(inputFilePath, outputFilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("createArchive: error transposing given file paths: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(outputFilePath), 0755); err != nil {
|
||||
return fmt.Errorf("createArchive: error creating output file path: %w", err)
|
||||
}
|
||||
|
||||
if err := compress(inputFilePath, outputFilePath, filepath.Dir(inputFilePath)); err != nil {
|
||||
return fmt.Errorf("createArchive: error creating archive: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func stripTrailingSlashes(path string) string {
|
||||
if len(path) > 0 && path[len(path)-1] == '/' {
|
||||
path = path[0 : len(path)-1]
|
||||
}
|
||||
|
||||
return path
|
||||
}
|
||||
|
||||
func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error) {
|
||||
inputFilePath, err := filepath.Abs(inputFilePath)
|
||||
if err == nil {
|
||||
outputFilePath, err = filepath.Abs(outputFilePath)
|
||||
}
|
||||
|
||||
return inputFilePath, outputFilePath, err
|
||||
}
|
||||
|
||||
func compress(inPath, outFilePath, subPath string) error {
|
||||
file, err := os.Create(outFilePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("compress: error creating out file: %w", err)
|
||||
}
|
||||
|
||||
prefix := path.Dir(outFilePath)
|
||||
gzipWriter := gzip.NewWriter(file)
|
||||
tarWriter := tar.NewWriter(gzipWriter)
|
||||
|
||||
var paths []string
|
||||
if err := filepath.WalkDir(inPath, func(path string, di fs.DirEntry, err error) error {
|
||||
paths = append(paths, path)
|
||||
return err
|
||||
}); err != nil {
|
||||
return fmt.Errorf("compress: error walking filesystem tree: %w", err)
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
if err := writeTarGz(p, tarWriter, prefix); err != nil {
|
||||
return fmt.Errorf("compress error writing %s to archive: %w", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
err = tarWriter.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("compress: error closing tar writer: %w", err)
|
||||
}
|
||||
|
||||
err = gzipWriter.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("compress: error closing gzip writer: %w", err)
|
||||
}
|
||||
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("compress: error closing file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeTarGz(path string, tarWriter *tar.Writer, prefix string) error {
|
||||
fileInfo, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writeTarGz: error getting file infor for %s: %w", path, err)
|
||||
}
|
||||
|
||||
if fileInfo.Mode()&os.ModeSocket == os.ModeSocket {
|
||||
return nil
|
||||
}
|
||||
|
||||
var link string
|
||||
if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
|
||||
var err error
|
||||
if link, err = os.Readlink(path); err != nil {
|
||||
return fmt.Errorf("writeTarGz: error resolving symlink %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(fileInfo, link)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writeTarGz: error getting file info header: %w", err)
|
||||
}
|
||||
header.Name = strings.TrimPrefix(path, prefix)
|
||||
|
||||
err = tarWriter.WriteHeader(header)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writeTarGz: error writing file info header: %w", err)
|
||||
}
|
||||
|
||||
if !fileInfo.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writeTarGz: error opening %s: %w", path, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = io.Copy(tarWriter, file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("writeTarGz: error copying %s to tar writer: %w", path, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
@ -23,7 +23,6 @@ import (
|
||||
"github.com/docker/docker/client"
|
||||
"github.com/kelseyhightower/envconfig"
|
||||
"github.com/leekchan/timeutil"
|
||||
"github.com/m90/targz"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/otiai10/copy"
|
||||
@ -364,7 +363,7 @@ func (s *script) takeBackup() error {
|
||||
s.logger.Infof("Removed tar file `%s`.", tarFile)
|
||||
return nil
|
||||
})
|
||||
if err := targz.Compress(backupSources, tarFile); err != nil {
|
||||
if err := createArchive(backupSources, tarFile); err != nil {
|
||||
return fmt.Errorf("takeBackup: error compressing backup folder: %w", err)
|
||||
}
|
||||
|
||||
|
1
go.mod
1
go.mod
@ -8,7 +8,6 @@ require (
|
||||
github.com/gofrs/flock v0.8.1
|
||||
github.com/kelseyhightower/envconfig v1.4.0
|
||||
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d
|
||||
github.com/m90/targz v0.0.0-20220208141135-d3baeef59a97
|
||||
github.com/minio/minio-go/v7 v7.0.16
|
||||
github.com/otiai10/copy v1.7.0
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
|
2
go.sum
2
go.sum
@ -450,8 +450,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d h1:2puqoOQwi3Ai1oznMOsFIbifm6kIfJaLLyYzWD4IzTs=
|
||||
github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d/go.mod h1:hO90vCP2x3exaSH58BIAowSKvV+0OsY21TtzuFGHON4=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/m90/targz v0.0.0-20220208141135-d3baeef59a97 h1:Uc/WzUKI/zvhkqIzk5TyaPE6AY1SD1DWGc7RV7cky4s=
|
||||
github.com/m90/targz v0.0.0-20220208141135-d3baeef59a97/go.mod h1:YZK3bSO/oVlk9G+v00BxgzxW2Us4p/R4ysHOBjk0fJI=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
|
@ -27,8 +27,8 @@ docker run --rm -it \
|
||||
-v compose_minio_backup_data:/minio_data \
|
||||
-v compose_webdav_backup_data:/webdav_data alpine \
|
||||
ash -c 'apk add gnupg && \
|
||||
echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /minio_data/backup/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db && \
|
||||
echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /webdav_data/data/my/new/path/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
|
||||
echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /minio_data/backup/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xvf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db && \
|
||||
echo 1234secret | gpg -d --pinentry-mode loopback --passphrase-fd 0 --yes /webdav_data/data/my/new/path/test-hostnametoken.tar.gz.gpg > /tmp/test-hostnametoken.tar.gz && tar -xvf /tmp/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db'
|
||||
|
||||
echo "[TEST:PASS] Found relevant files in decrypted and untared remote backups."
|
||||
|
||||
|
1
test/ownership/.gitignore
vendored
Normal file
1
test/ownership/.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
local
|
27
test/ownership/docker-compose.yml
Normal file
27
test/ownership/docker-compose.yml
Normal file
@ -0,0 +1,27 @@
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:14-alpine
|
||||
restart: unless-stopped
|
||||
labels:
|
||||
- docker-volume-backup.stop-during-backup=true
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
environment:
|
||||
- POSTGRES_PASSWORD=1FHJMSwt0yhIN1zS7I4DilGUhThBKq0x
|
||||
- POSTGRES_USER=test
|
||||
- POSTGRES_DB=test
|
||||
|
||||
backup:
|
||||
image: offen/docker-volume-backup:${TEST_VERSION}
|
||||
restart: always
|
||||
environment:
|
||||
BACKUP_FILENAME: backup.tar.gz
|
||||
volumes:
|
||||
- postgres_data:/backup/postgres:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- ./local:/archive
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
28
test/ownership/run.sh
Normal file
28
test/ownership/run.sh
Normal file
@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
# This test refers to https://github.com/offen/docker-volume-backup/issues/71
|
||||
|
||||
set -e
|
||||
|
||||
cd $(dirname $0)
|
||||
|
||||
mkdir -p local
|
||||
|
||||
docker-compose up -d
|
||||
sleep 5
|
||||
|
||||
docker-compose exec backup backup
|
||||
|
||||
sudo tar --same-owner -xvf ./local/backup.tar.gz -C /tmp
|
||||
|
||||
sudo find /tmp/backup/postgres > /dev/null
|
||||
echo "[TEST:PASS] Backup contains files at expected location"
|
||||
|
||||
for file in $(sudo find /tmp/backup/postgres); do
|
||||
if [ "$(sudo stat -c '%u:%g' $file)" != "70:70" ]; then
|
||||
echo "[TEST:FAIL] Unexpected file ownership for $file: $(sudo stat -c '%u:%g' $file)"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "[TEST:PASS] All files and directories in backup preserved their ownership."
|
||||
|
||||
docker-compose down --volumes
|
@ -49,7 +49,7 @@ services:
|
||||
condition: on-failure
|
||||
|
||||
pg:
|
||||
image: postgres:12.2-alpine
|
||||
image: postgres:14-alpine
|
||||
environment:
|
||||
POSTGRES_PASSWORD: example
|
||||
labels:
|
||||
|
Loading…
Reference in New Issue
Block a user