Merged fork/feature/duckdb branch

Author: proofrock

.gitignore

@@ -4,6 +4,7 @@ bin/
 .idea/
 /notes.md
 test/*.db
+test/*.db.wal
 
 # Created by https://www.toptal.com/developers/gitignore/api/go,macos,linux,windows
 # Edit at https://www.toptal.com/developers/gitignore?templates=go,macos,linux,windows

README.md

@@ -2,14 +2,14 @@
 
 > I recently started a [discussion](https://github.com/proofrock/ws4sql/discussions/44) over the future direction for this project. Take a look, and chip in if you want!
 
-**`ws4sql`** is a server application that, applied to one or more sqlite files, allows to perform SQL queries and statements on them via REST (or better, JSON over HTTP).
+**`ws4sql`** is a server application that, applied to one or more database files, allows to perform SQL queries and statements on them via REST (or better, JSON over HTTP). It supports SQLite and DuckDB.
 
-Possible use cases are the ones where remote access to a sqlite db is useful/needed, for example a data layer for a remote application, possibly serverless or even called from a web page (*after security considerations* of course).
+Possible use cases are the ones where remote access to an embedded db is useful/needed, for example a data layer for a remote application, possibly serverless or even called from a web page (*after security considerations* of course).
 
-Client libraries are available, that will abstract the "raw" JSON-based communication. See 
+Client libraries are available, that will abstract the "raw" JSON-based communication. See
 [here](https://github.com/proofrock/ws4sqlite-client-jvm) for Java/JVM, [here](https://github.com/proofrock/ws4sqlite-client-go) for Go(lang); others will follow.
 
-As a quick example, after launching 
+As a quick example, after launching
 
 ```bash
 ws4sql --db mydatabase.db
@@ -56,7 +56,7 @@ Obtaining an answer of
 
 [Docs](https://germ.gitbook.io/ws4sql/), a [Tutorial](https://germ.gitbook.io/ws4sql/tutorial), a [Discord](https://discord.gg/nBCcq2VQPu).
 
-- Aligned to [**SQLite 3.45.1**](https://sqlite.org/releaselog/3_45_1.html);
+- Aligned to [**SQLite 3.46.1**](https://sqlite.org/releaselog/3_46_1.html) and [**DuckDB 1.1.3**](https://github.com/duckdb/duckdb/releases/tag/v1.1.3);
 - A [**single executable file**](https://germ.gitbook.io/ws4sql/documentation/installation) (written in Go);
 - HTTP/JSON access, with [**client libraries**](https://germ.gitbook.io/ws4sql/client-libraries) for convenience;
 - Directly call `ws4sql` on a database (as above), many options available using a YAML companion file;
@@ -72,7 +72,7 @@ Obtaining an answer of
 - [**Scheduled tasks**](https://germ.gitbook.io/ws4sql/documentation/sched_tasks), cron-like and/or at startup, also configurable per-db;
 - Scheduled tasks can be: backup (with rotation), vacuum and/or a set of SQL statements;
 - Provide [**initialization statements**](https://germ.gitbook.io/ws4sql/documentation/configuration-file#initstatements) to execute when a DB is created;
-- [**WAL**](https://sqlite.org/wal.html) mode enabled by default, can be [disabled](https://germ.gitbook.io/ws4sql/documentation/configuration-file#disablewalmode);
+- (for SQLite) [**WAL**](https://sqlite.org/wal.html) mode enabled by default, can be [disabled](https://germ.gitbook.io/ws4sql/documentation/configuration-file#disablewalmode);
 - [**Quite fast**](features/performances.md)!
 - [**Embedded web server**](https://germ.gitbook.io/ws4sql/documentation/web-server) to directly serve web pages that can access ws4sql without CORS;
 - Compact codebase;
@@ -82,22 +82,22 @@ Obtaining an answer of
 
 # Security Features
 
-* [**Authentication**](documentation/security.md#authentication) can be configured
-  * on the client, either using HTTP Basic Authentication or specifying the credentials in the request;
-  * on the server, either by specifying credentials (also with hashed passwords) or providing a query to look them up in the db itself;
-  * customizable `Not Authorized` error code (if 401 is not optimal)
-* A database can be opened in [**read-only mode**](documentation/security.md#read-only-databases) (only queries will be allowed);
-* It's possible to enforce using [**only stored statements**](documentation/security.md#stored-statements-to-prevent-sql-injection), to avoid some forms of SQL injection and receiving SQL from the client altogether;
-* [**CORS Allowed Origin**](documentation/security.md#cors-allowed-origin) can be configured and enforced;
-* It's possible to [**bind**](documentation/security.md#binding-to-a-network-interface) to a network interface, to limit access.
+- [**Authentication**](documentation/security.md#authentication) can be configured
+  - on the client, either using HTTP Basic Authentication or specifying the credentials in the request;
+  - on the server, either by specifying credentials (also with hashed passwords) or providing a query to look them up in the db itself;
+  - customizable `Not Authorized` error code (if 401 is not optimal)
+- A database can be opened in [**read-only mode**](documentation/security.md#read-only-databases) (only queries will be allowed);
+- It's possible to enforce using [**only stored statements**](documentation/security.md#stored-statements-to-prevent-sql-injection), to avoid some forms of SQL injection and receiving SQL from the client altogether;
+- [**CORS Allowed Origin**](documentation/security.md#cors-allowed-origin) can be configured and enforced;
+- It's possible to [**bind**](documentation/security.md#binding-to-a-network-interface) to a network interface, to limit access.
 
 # Design Choices
 
 Some design choices:
 
-* Very thin layer over SQLite. Errors and type translation, for example, are those provided by the SQLite driver;
-* Doesn't include HTTPS, as this can be done easily (and much more securely) with a [reverse proxy](documentation/security.md#use-a-reverse-proxy-if-going-on-the-internet);
-* Doesn't support SQLite extensions, to improve portability.
+- Very thin layer over the database. Errors and type translation, for example, are those provided by the db driver;
+- Doesn't include HTTPS, as this can be done easily (and much more securely) with a [reverse proxy](documentation/security.md#use-a-reverse-proxy-if-going-on-the-internet);
+- Doesn't support SQLite/DuckDB extensions, to improve portability.
 
 # Contacts and Support
 
@@ -112,7 +112,8 @@ Many thanks and all the credits to these awesome projects:
 - [gofiber's fiber](https://github.com/gofiber/fiber) (MIT License);
 - [klauspost's compress](https://github.com/klauspost/compress) (3-Clause BSD license);
 - [mitchellh's go-homedir](https://github.com/mitchellh/go-homedir) (MIT License);
-- [modernc.org's sqlite](https://gitlab.com/cznic/sqlite) (3-Clause BSD License);
+- [mattn's go-sqlite3](https://github.com/mattn/go-sqlite3) (MIT License);
+- [modernc.org's go-duckdb](https://github.com/marcboeker/go-duckdb) (MIT License);
 - [wI2L's jettison](https://github.com/wI2L/jettison) (MIT License)
 - and of course, [Google Go](https://go.dev).
 
@@ -124,4 +125,4 @@ Import my [key](https://public.germanorizzo.it/4BD993A579025E4932C0110DC368E8BA7
 
 ```bash
 curl -sSL https://public.germanorizzo.it/4BD993A579025E4932C0110DC368E8BA7D4453F6.gpgkey | gpg --import -
-```
+```

ROAD_TO_WS4SQL.md

@@ -5,27 +5,35 @@ The version in this branch is a work in progress to slowly add features and (unf
 # Changes
 
 - SQLite is embedded via [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3) and CGO. Should be way faster.
+- Support for DuckDB (see below).
 - Target platforms (because of CGO) are now 6 (`win/amd64`, `macos/amd64`, `macos/arm64`, `linux/amd64`, `linux/arm64`, `linux/arm6`).
-- [**BREAKING**] When running the app, the config files must be specified on the command line, the file paths cannot be used anymore (there). This is described in the "Migration" section below. The file path is in the config file.
+- [**BREAKING CHANGE**] When running the app, the config files must be specified on the command line, the file paths cannot be used anymore (there). This is described in the "Migration" section below. The file path is in the config file.
 - The only exception is a "simple case" to serve a file path without any config. This can be done with the new `--quick-db` parameter.
+- Fail fast if the request is empty, don't even attempt to authenticate
 
 # Migration
 
 - For any `--db` and `--mem-db` switch that was used, an explicit YAML config file must be created. The format is the same, but there is a new section at the beginning:
+
 ```yaml
 database:
-  type: SQLITE          # Only SQLITE for now. If omitted, defaults to SQLITE      
-  inMemory: false       # If type = SQLITE. The db is a memory one? If omitted, defaults to false
-  path: ".../test.db"   # If type = SQLITE. The db file path.
-  id: test              # If omitted and !inMemory, calculates it from the file name (if type = SQLITE)
+  type: SQLITE          # SQLITE or DUCKDB. If omitted, defaults to SQLITE      
+  inMemory: false       # If type = SQLITE|DUCKDB. The db is a memory one? If omitted, defaults to false
+  path: ".../test.db"   # If type = SQLITE|DUCKDB. The db file path.
+  id: test              # If omitted and !inMemory, calculates it from the file name (if type = SQLITE|DUCKDB)
   disableWALMode: false # If type = SQLITE. Same as before, but moved here.
   readOnly: false       # Same as before, but moved here.
 ```
 
+# Specific to DuckDB
+
+- `noFail` is not supported.
+- Placeholders for named parameters are `$VAL`, not `:VAL` as in SQLite.
+- As DuckDB does not support read-only transactions, when `readOnly` is specified the requests won't be processed in a transaction.
+
 # Roadmap
 
 1. Support mariadb/mysql
-1. Support duckdb (and iron out all the incompatibilities)
 1. Support postgresql
 1. ...
-1. Profit!
+1. Profit!

src/authentication.go

@@ -27,6 +27,8 @@ import (
 	"strings"
 
 	mllog "github.com/proofrock/go-mylittlelogger"
+	"github.com/proofrock/ws4sql/structs"
+	"github.com/proofrock/ws4sql/utils"
 )
 
 const (
@@ -38,11 +40,11 @@ const (
 // Version with explicit credentials, called by the authentication
 // middleware and by the "other" auth function, that accepts
 // a request.
-func applyAuthCreds(db *db, user, password string) error {
+func applyAuthCreds(db *structs.Db, user, password string) error {
 	if db.Auth.ByQuery != "" {
 		// Auth via query. Looks into the database for the credentials;
 		// needs a query that is correctly parametrized.
-		nameds := vals2nameds(map[string]interface{}{"user": user, "password": password})
+		nameds := utils.Vals2nameds(map[string]interface{}{"user": user, "password": password})
 		row := db.DbConn.QueryRowContext(context.Background(), db.Auth.ByQuery, nameds...)
 		var foo interface{}
 		if err := row.Scan(&foo); err == sql.ErrNoRows {
@@ -65,7 +67,7 @@ func applyAuthCreds(db *db, user, password string) error {
 // Checks auth. If auth is granted, returns nil, if not an error.
 // Version with request, extracts the credentials from the request
 // (when authmode = INLINE) and delegates to applyAuthCreds()
-func applyAuth(db *db, req *request) error {
+func applyAuth(db *structs.Db, req *structs.Request) error {
 	if req.Credentials == nil {
 		return errors.New("missing auth credentials")
 	}
@@ -74,7 +76,7 @@ func applyAuth(db *db, req *request) error {
 
 // Parses the authentication configurations. Builds a few structures,
 // should be pretty straightforward to read.
-func parseAuth(db *db) {
+func parseAuth(db *structs.Db) {
 	auth := *db.Auth
 	if strings.ToUpper(auth.Mode) != authModeInline && strings.ToUpper(auth.Mode) != authModeHttp {
 		mllog.Fatal("Auth Mode must be INLINE or HTTP")