Backup feature for DuckDB databases

Author: proofrock

.gitignore

@@ -1,5 +1,6 @@
 src/ws4sql
 bin/
+env/
 .vscode/
 .idea/
 /notes.md

Makefile

@@ -10,7 +10,7 @@ upd-libraries:
 	cd src; go mod tidy
 
 test:
-	cd src; go test -v -timeout 6m
+	cd src; go test -v -timeout 8m
 
 build-prepare:
 	make cleanup

ROAD_TO_WS4SQL.md

@@ -1,8 +1,10 @@
+# Road to `ws4sql`
+
 The next version of `ws4sqlite` will be called `ws4sql`, because it will support more RDBMs than sqlite.
 
 The version in this branch is a work in progress to add features and (unfortunately) breaking changes; here is a review of what changed compared to `ws4sqlite` "stable" + a migration path.
 
-# Changes
+## Changes
 
 - SQLite is embedded via [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and CGO. Should be way faster.
 - Support for DuckDB (see below).
@@ -14,7 +16,7 @@ The version in this branch is a work in progress to add features and (unfortunat
 - Docker images are now based on `distroless/static-debian12`.
 - Docker images are now hosted on Github's Container Registry.
 
-# Migration
+## Migration
 
 - For any `--db` and `--mem-db` switch that was used, an explicit YAML config file must be created. The format is the same, but there is a new section at the beginning:
 
@@ -28,13 +30,15 @@ database:
   readOnly: false       # Same as before, but moved here.
 ```
 
-# Specific to DuckDB
+## Specific to DuckDB
 
 - `noFail` is not supported.
 - Placeholders for named parameters are `$VAL`, not `:VAL` as in SQLite.
 - As DuckDB does not support read-only transactions, when `readOnly` is specified the requests won't be processed in a transaction.
+- Duckdb exports backups in a folder. A backup is performed by exporting to a temp folder and zipping it. CSV format is used (`EXPORT DATABASE '...' (FORMAT CSV)`).
+- At least for now, when instructed, a `VACUUM` is performed, not a `VACUUM ANALYZE`.
 
-# Roadmap
+## Roadmap
 
 1. Support mariadb/mysql
 1. Support postgresql

src/engines/duckdb.go

@@ -17,8 +17,10 @@
 package engines
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -152,3 +154,24 @@ func (s *duckdbEngine) SanitizeResponseField(fldVal interface{}) (interface{}, e
 		return fldVal, nil
 	}
 }
+
+// Saves a backup to a temporary folder and zips it
+func (s *duckdbEngine) DoBackup(task structs.ScheduledTask, fname string, now string) error {
+	tempDir, err := os.MkdirTemp("", fmt.Sprintf("duckdb_backup_%s_%s", *task.Db.DatabaseDef.Id, now))
+	if err != nil {
+		return fmt.Errorf("sched. task (backup prep): %w", err)
+	}
+	defer os.RemoveAll(tempDir) // Clean up after function returns
+
+	backupPath := strings.ReplaceAll(filepath.Join(tempDir, "backup"), "'", "''")
+	_, err = task.Db.DbConn.ExecContext(context.Background(), fmt.Sprintf("EXPORT DATABASE '%s' (FORMAT CSV)", backupPath))
+	if err != nil {
+		return fmt.Errorf("sched. task (backup): %w", err)
+	}
+
+	if err := utils.ZipFolder(backupPath, fname); err != nil {
+		return fmt.Errorf("sched. task (backup zip): %w", err)
+	}
+
+	return nil
+}

src/engines/interface.go

@@ -31,6 +31,7 @@ type Engine interface {
 	CheckConfig(dbConfig structs.Db) structs.Db
 	CheckRequest(body structs.Request) *structs.WsError
 	SanitizeResponseField(fldVal interface{}) (interface{}, error)
+	DoBackup(task structs.ScheduledTask, fname string, now string) error
 }
 
 const ID_SQLITE = "SQLITE"

src/engines/sqlite.go

@@ -17,7 +17,9 @@
 package engines
 
 import (
+	"context"
 	"database/sql"
+	"fmt"
 	"path/filepath"
 	"strings"
 
@@ -108,3 +110,15 @@ func (s *sqliteEngine) CheckConfig(dbConfig structs.Db) structs.Db {
 func (s *sqliteEngine) SanitizeResponseField(fldVal interface{}) (interface{}, error) {
 	return fldVal, nil
 }
+
+func (s *sqliteEngine) DoBackup(task structs.ScheduledTask, fname string, _ string) error {
+	stat, err := task.Db.DbConn.PrepareContext(context.Background(), "VACUUM INTO ?")
+	if err != nil {
+		return fmt.Errorf("sched. task (backup prep): %s", err.Error())
+	}
+	defer stat.Close()
+	if _, err := stat.Exec(fname); err != nil {
+		return fmt.Errorf("sched. task (backup): %s", err.Error())
+	}
+	return nil
+}

src/sched_tasks.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/mitchellh/go-homedir"
 	mllog "github.com/proofrock/go-mylittlelogger"
+	"github.com/proofrock/ws4sql/engines"
 	"github.com/proofrock/ws4sql/structs"
 
 	cronDesc "github.com/lnquy/cron"
@@ -37,7 +38,7 @@ import (
 const bkpTimeFormat = "060102-1504"
 
 // Used when deleting older backup files, the date/time is substituted with '?'
-var bkpTimeGlob = strings.Repeat("?", len(bkpTimeFormat))
+const bkpTimeGlob = "??????-????"
 
 // Parses a backup plan, checks that it is well-formed and returns a function that
 // will be called by cron and executes the plan.
@@ -81,7 +82,7 @@ func doTask(task structs.ScheduledTask) func() {
 		task.Db.Mutex.Lock()
 		defer task.Db.Mutex.Unlock()
 
-		if task.DoVacuum {
+		if task.DoVacuum { // SQLite only
 			if _, err := task.Db.DbConn.ExecContext(context.Background(), "VACUUM"); err != nil {
 				mllog.Error("sched. task (vacuum): ", err.Error())
 				return
@@ -91,14 +92,9 @@ func doTask(task structs.ScheduledTask) func() {
 		if task.DoBackup {
 			now := time.Now().Format(bkpTimeFormat)
 			fname := fmt.Sprintf(filepath.Join(bkpDir, bkpFile), now)
-			stat, err := task.Db.DbConn.PrepareContext(context.Background(), "VACUUM INTO ?")
+			err := engines.GetFlavorForDb(*task.Db).DoBackup(task, fname, now)
 			if err != nil {
-				mllog.Error("sched. task (backup prep): ", err.Error())
-				return
-			}
-			defer stat.Close()
-			if _, err := stat.Exec(fname); err != nil {
-				mllog.Error("sched. task (backup): ", err.Error())
+				mllog.Error(err.Error())
 				return
 			}
 			// delete the backup files, except for the last n

src/sched_tasks_test.go

@@ -156,6 +156,110 @@ func TestSchedTasks(t *testing.T) {
 	time.Sleep(time.Second)
 }
 
+// For DuckDB. Two minutes too.
+func TestDDBSchedTasks(t *testing.T) {
+	defer os.Remove("../test/test1.zip")
+	defer os.Remove("../test/test2.zip")
+	defer Shutdown()
+
+	sched := "* * * * *"
+
+	cfg := structs.Config{
+		Bindhost: "0.0.0.0",
+		Port:     12321,
+		Databases: []structs.Db{
+			{
+				DatabaseDef: structs.DatabaseDef{
+					Type: utils.Ptr("DUCKDB"),
+					Id:   utils.Ptr("test1"),
+					Path: utils.Ptr("../test/test1.db"),
+				},
+				Maintenance: &structs.ScheduledTask{
+					Schedule:       &sched,
+					DoVacuum:       false,
+					DoBackup:       true,
+					BackupTemplate: "../test/test1_%s.zip",
+					NumFiles:       1,
+				},
+			}, {
+				DatabaseDef: structs.DatabaseDef{
+					Type: utils.Ptr("DUCKDB"),
+					Id:   utils.Ptr("test2"),
+					Path: utils.Ptr("../test/test2.db"),
+				},
+				Maintenance: &structs.ScheduledTask{
+					Schedule:       &sched,
+					DoVacuum:       false,
+					DoBackup:       true,
+					BackupTemplate: "../test/test2_%s.zip",
+					NumFiles:       1,
+				},
+			},
+		},
+	}
+
+	cleanSchedTasksFiles(cfg)
+	defer cleanSchedTasksFiles(cfg)
+
+	go launch(cfg, true)
+
+	time.Sleep(time.Second)
+
+	if !utils.FileExists(*cfg.Databases[0].DatabaseDef.Path) || !utils.FileExists(*cfg.Databases[1].DatabaseDef.Path) {
+		t.Error("db file not created")
+		return
+	}
+
+	req := structs.Request{
+		Transaction: []structs.RequestItem{
+			{
+				Statement: "CREATE TABLE T1 (ID INT PRIMARY KEY, VAL TEXT NOT NULL)",
+			},
+		},
+	}
+	code, _, _ := call("test1", req, t)
+	if code != 200 {
+		t.Error("did not succeed")
+		return
+	}
+
+	time.Sleep(time.Minute)
+
+	now := time.Now().Format(bkpTimeFormat)
+	bk1 := fmt.Sprintf(cfg.Databases[0].Maintenance.BackupTemplate, now)
+	bk2 := fmt.Sprintf(cfg.Databases[1].Maintenance.BackupTemplate, now)
+
+	if !utils.FileExists(bk1) || !utils.FileExists(bk2) {
+		t.Error("backup file not created")
+		return
+	}
+
+	stat1, _ := os.Stat(bk1)
+	stat2, _ := os.Stat(bk2)
+
+	if stat2.Size() >= stat1.Size() {
+		t.Error("backup files sizes are inconsistent")
+	}
+
+	time.Sleep(time.Minute)
+
+	now = time.Now().Format(bkpTimeFormat)
+	bk3 := fmt.Sprintf(cfg.Databases[0].Maintenance.BackupTemplate, now)
+	bk4 := fmt.Sprintf(cfg.Databases[1].Maintenance.BackupTemplate, now)
+
+	if !utils.FileExists(bk3) || !utils.FileExists(bk4) {
+		t.Error("backup file not created, the second time")
+		return
+	}
+
+	if utils.FileExists(bk1) || utils.FileExists(bk2) {
+		t.Error("backup file not rotated")
+		return
+	}
+
+	time.Sleep(time.Second)
+}
+
 // Takes one minute
 func TestSchedTasksWithReadOnly(t *testing.T) {
 	defer os.Remove("../test/test.db")

src/utils/utils.go

@@ -17,10 +17,13 @@
 package utils
 
 import (
+	"archive/zip"
 	"database/sql"
 	"encoding/json"
 	"errors"
+	"io"
 	"os"
+	"path/filepath"
 	"slices"
 	"strings"
 
@@ -122,3 +125,44 @@ func GetDefault[T any](m orderedmap.OrderedMap, key string) T {
 
 	return value.(T)
 }
+
+// Zips a folder to a path
+func ZipFolder(srcDir, destZip string) error {
+	zipFile, err := os.Create(destZip)
+	if err != nil {
+		return err
+	}
+	defer zipFile.Close()
+
+	zipWriter := zip.NewWriter(zipFile)
+	defer zipWriter.Close()
+
+	return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		relPath, err := filepath.Rel(srcDir, path)
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		file, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		zipEntry, err := zipWriter.Create(relPath)
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(zipEntry, file)
+		return err
+	})
+}