From 80c9983bb917c49682cd83f8621a891797982591 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 25 Oct 2026 09:34:17 +0600 Subject: [PATCH 1/2] chore: label varmq v1.10.0, neutral dashboard descriptions The three go.mod files require varmq v1.10.0 and keep building the unreleased local checkout through the replace directive until the tag exists. The redis and sqlite dataset descriptions list the libraries alphabetically and call C1/C4 library instances in one process, which is what the harness runs. pool:bench gets -timeout 0 like the full run. Co-Authored-By: Claude Opus 5.5 --- Taskfile.yml | 8 ++++---- pool/Taskfile.yml | 2 +- pool/go.mod | 4 +--- redis/go.mod | 2 +- sqlite/go.mod | 2 +- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index e27303d..4af6d0a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -10,8 +10,8 @@ vars: DATA: worker/public/data DATA_URL: https://benchmarks.goptics.org/dataset BAR: vizb bar -p /n/y/x -T s -M MB --round -l --border-radius 8,8,0,0 - # C1, Enqueue and Latency groups race VarMQ's persistent queue against these; - # VarMQDistributed only appears in the C4 groups. Keep in step with redis/subjects.go. + # Subjects shown in the C1, Enqueue and Latency groups; VarMQDistributed only + # appears in the C4 groups. Keep in step with redis/subjects.go. SUBJECTS: '(VarMQ|Asynq|Rmq|Taskq|GocraftWork)' # catalog order = dashboard order; listed literally (a glob sorts 4cpu before 24cpu) JSONS: >- @@ -32,13 +32,13 @@ tasks: - '{{.BAR}} --stack pool/bench/pool-4cpu.txt -n "Worker Pool (4 CPUs)" --id pool-4cpu -d "Same matrix at GOMAXPROCS 4" -o {{.DATA}}/pool-4cpu.json' - >- {{.BAR}} --stack redis/bench/redis.txt -n "Message Queue Redis" --id redis - -d "VarMQ (redisq) vs Asynq, rmq, taskq, gocraft/work on Redis 7.2. C1 groups: one consumer process; C4: four. Plus enqueue-only and pickup latency" + -d "Asynq, gocraft/work, rmq, taskq and VarMQ (redisq) on Redis 7.2. C1 groups: one library instance; C4: four instances in the same process. Plus enqueue-only and enqueue-to-handler latency" -f '^(Benchmark)?((Durable/[^/]+C1|Enqueue/Enqueue|Latency/[^/]+)/[^/]+/{{.SUBJECTS}}$|Durable/[^/]+C4/)' --better p50-ms:low,p99-ms:low,max-ms:low -o {{.DATA}}/redis.json - >- {{.BAR}} --stack sqlite/bench/sqlite.txt -n "Message Queue SQLite" --id sqlite - -d "VarMQ (sqliteq) vs goqite, backlite, River on SQLite: one queue, drain, priority, ten queues; WAL checkpoint 1000 vs 10000 pages" + -d "backlite, goqite, River and VarMQ (sqliteq) on SQLite: one queue, drain, priority, ten queues; WAL checkpoint 1000 vs 10000 pages" --better walMB:low -o {{.DATA}}/sqlite.json - jq -se 'all(.data | length > 0)' {{.JSONS}} - jq -s 'map({id, name})' {{.JSONS}} > {{.DATA}}/_catalog.json diff --git a/pool/Taskfile.yml b/pool/Taskfile.yml index 9cbd2a5..5c6a903 100644 --- a/pool/Taskfile.yml +++ b/pool/Taskfile.yml @@ -8,7 +8,7 @@ tasks: bench: desc: Every benchmark once cmds: - - go test -run xxx -bench . -benchtime=1x -benchmem + - go test -run xxx -bench . -benchtime=1x -benchmem -timeout 0 txt: desc: Full matrix at 24, 8 and 4 CPUs into bench/ (hours) cmds: diff --git a/pool/go.mod b/pool/go.mod index 24a2c5f..bede760 100644 --- a/pool/go.mod +++ b/pool/go.mod @@ -6,7 +6,7 @@ require ( github.com/alitto/pond/v2 v2.7.2 github.com/bytedance/gopkg v0.1.4 github.com/gammazero/workerpool v1.2.2 - github.com/goptics/varmq v1.9.0 + github.com/goptics/varmq v1.10.0 github.com/panjf2000/ants/v2 v2.12.1 github.com/sourcegraph/conc v0.3.0 golang.org/x/sync v0.24.0 @@ -18,6 +18,4 @@ require ( go.uber.org/multierr v1.9.0 // indirect ) -// replace github.com/goptics/varmq => ../varmq - replace github.com/goptics/varmq => ../../varmq diff --git a/redis/go.mod b/redis/go.mod index 60cfc2c..2d21eb4 100644 --- a/redis/go.mod +++ b/redis/go.mod @@ -8,7 +8,7 @@ require ( github.com/gocraft/work v0.5.1 github.com/gomodule/redigo v1.9.3 github.com/goptics/redisq v1.5.0 - github.com/goptics/varmq v1.9.0 + github.com/goptics/varmq v1.10.0 github.com/hibiken/asynq v0.26.0 github.com/redis/go-redis/v9 v9.23.0 github.com/vmihailenco/taskq/v3 v3.2.9 diff --git a/sqlite/go.mod b/sqlite/go.mod index 613bada..7d8cd4d 100644 --- a/sqlite/go.mod +++ b/sqlite/go.mod @@ -4,7 +4,7 @@ go 1.26.0 require ( github.com/goptics/sqliteq v0.3.0 - github.com/goptics/varmq v1.9.0 + github.com/goptics/varmq v1.10.0 github.com/mattn/go-sqlite3 v1.14.28 github.com/mikestefanello/backlite v0.8.0 github.com/riverqueue/river v0.49.0 From 9b8f68e4734b82249a565c1a5a809d829c760ef8 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 25 Oct 2026 09:51:18 +0600 Subject: [PATCH 2/2] docs: neutral READMEs, every library linked, workloads explained - Root README: no results, no VarMQ framing; suites, environment, a linked versions table and how to run. - Every library and client is linked to its repository; varmq is labelled v1.10.0 (built from the local checkout until it is tagged). - pool: what a producer is, what each task does, what is timed, how each subject submits and tears down. - redis: a Workloads section: what one op does, the payload, producers, W and C (library instances in one process, not processes), the enqueue-only and enqueue-to-handler latency runs, and what fails a row. - sqlite: per-queue job counts, stored payloads, file layout, priority mapping, what drain timing includes, walMB, the full failure list. - Known harness gaps are stated where they apply. Co-Authored-By: Claude Opus 5.5 --- README.md | 71 +++++++++++++++++++------------------- pool/README.md | 79 +++++++++++++++++++++++------------------- redis/NOTES.md | 4 +-- redis/README.md | 90 +++++++++++++++++++++++++++++------------------- sqlite/README.md | 77 +++++++++++++++++++++++++---------------- 5 files changed, 184 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index b746818..a75f96e 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,50 @@ -# VarMQ Benchmarks +# Benchmarks [![Live charts](https://img.shields.io/badge/%F0%9F%93%8A-Live_Charts-blue?style=for-the-badge)](https://benchmarks.goptics.org) -Benchmark suites for [VarMQ](https://github.com/goptics/varmq) against other Go worker pools and message queues. Raw `go test -bench` output is committed under `/bench/`; the charts at are built from those files. +Benchmarks of Go worker pools and job queues: in-process worker pools, Redis-backed queues and SQLite-backed queues. Raw `go test -bench` output is committed under `/bench/`, and the charts at are built from it. ## Suites -| Suite | Compares | Charts | Details | -|-------|----------|--------|---------| -| Worker pool | VarMQ vs Pond, Ants, WorkerPool, Conc, ErrGroup, Gopool and two channel baselines | [24](https://benchmarks.goptics.org/?id=pool-24cpu), [8](https://benchmarks.goptics.org/?id=pool-8cpu), [4 CPUs](https://benchmarks.goptics.org/?id=pool-4cpu) | [pool/README.md](pool/README.md) | -| Message queue, Redis | VarMQ (redisq) vs Asynq, rmq, taskq, gocraft/work, with one and four consumer processes | [redis](https://benchmarks.goptics.org/?id=redis) | [redis/README.md](redis/README.md) | -| Message queue, SQLite | VarMQ (sqliteq) vs goqite, backlite, River, at WAL checkpoint 1000 and 10000 pages | [sqlite](https://benchmarks.goptics.org/?id=sqlite) | [sqlite/README.md](sqlite/README.md) | - -## Results - -Pool run of 2026-10-24, Redis and SQLite runs of 2026-10-25, all `-benchtime=2x`. Full tables in the suite READMEs. - -- Pool: VarMQ is the fastest subject in 40, 36 and 20 of the 55 rows at 24, 8 and 4 CPUs. It is slower than the buffered-channel baseline on the one-worker `Loop10` rows, except `1Mu-1t`, where the two are level. -- Redis: with the `Noop` task, VarMQ is first in 2 of the 6 one-process rows and VarMQDistributed in 2 of the 6 four-process rows, all at 100 producers; rmq leads every single-producer row. Pickup latency p50: taskq 0.03876 ms, VarMQDistributed 0.06492 ms, VarMQ 0.09407 ms, rmq 1.464 ms, Asynq 979.7 ms. -- SQLite: VarMQ (sqliteq) is first in all 8 scenario and checkpoint groups: on `Mixed1Q` at ckpt1000 it runs 3.1x goqite, 4.1x River and 23x backlite, and ten queues in separate files reach 107,459 jobs/s at ckpt10000. +| Suite | What runs | Libraries | Charts | Details | +|-------|-----------|-----------|--------|---------| +| Worker pool | 9 subjects (7 libraries plus two channel baselines); 5 workload shapes, 4 tasks, 4 pool sizes, 1M tasks per op; recorded at GOMAXPROCS 24, 8 and 4 | [ants](https://github.com/panjf2000/ants), [conc](https://github.com/sourcegraph/conc), errgroup ([x/sync](https://github.com/golang/sync)), gopool ([bytedance/gopkg](https://github.com/bytedance/gopkg)), [pond](https://github.com/alitto/pond), [VarMQ](https://github.com/goptics/varmq), [workerpool](https://github.com/gammazero/workerpool) | [pool-24cpu](https://benchmarks.goptics.org/?id=pool-24cpu), [pool-8cpu](https://benchmarks.goptics.org/?id=pool-8cpu), [pool-4cpu](https://benchmarks.goptics.org/?id=pool-4cpu) | [pool/README.md](pool/README.md) | +| Message queue, Redis | 6 subjects from 5 libraries; 100,000 jobs per op through 1 or 4 library instances in one process, plus enqueue-only and enqueue-to-handler latency | [Asynq](https://github.com/hibiken/asynq), [gocraft/work](https://github.com/gocraft/work), [rmq](https://github.com/adjust/rmq), [taskq](https://github.com/vmihailenco/taskq), [VarMQ](https://github.com/goptics/varmq) with [redisq](https://github.com/goptics/redisq) | [redis](https://benchmarks.goptics.org/?id=redis) | [redis/README.md](redis/README.md) | +| Message queue, SQLite | 5 subjects from 4 libraries; 4 scenarios at WAL checkpoint 1000 and 10000 pages | [backlite](https://github.com/mikestefanello/backlite), [goqite](https://github.com/maragudk/goqite), [River](https://github.com/riverqueue/river), [VarMQ](https://github.com/goptics/varmq) with [sqliteq](https://github.com/goptics/sqliteq) | [sqlite](https://benchmarks.goptics.org/?id=sqlite) | [sqlite/README.md](sqlite/README.md) | ## Environment -Intel Core i7-13700 (24 threads), linux/amd64, Go 1.26.2. Pool suite at GOMAXPROCS 24, 8 and 4. Redis 7.2 (Debian image) pinned to CPU 14, client on cores 0-13 and 16-23. SQLite databases on ext4 over NVMe. - -| Library | Version | -|---------|---------| -| varmq | 415ad73 (unreleased, `replace` in each go.mod) | -| redisq | v1.5.0 | -| sqliteq | v0.3.0 | -| pond | v2.7.2 | -| ants | v2.12.1 | -| workerpool | v1.2.2 | -| conc | v0.3.0 | -| x/sync (errgroup) | v0.24.0 | -| bytedance/gopkg (gopool) | v0.1.4 | -| asynq | v0.26.0 | -| rmq | v5.2.0 | -| taskq | v3.2.9 | -| gocraft/work | v0.5.1 | -| goqite | v0.4.0 | -| backlite | v0.8.0 | -| river | v0.49.0 | +Intel Core i7-13700 (24 threads), linux/amd64, Go 1.26, SQLite databases on ext4 over NVMe. + +The pool suite sets GOMAXPROCS to 24, 8 and 4 with no core pinning. The SQLite suite runs on all 24 threads. Redis 7.2 (Debian image) is pinned to CPU 14 and the Redis suite's client to CPUs 0-13 and 16-23 (22 threads), so Redis has a whole physical core. + +| Library | Version | Suite | +|---------|---------|-------| +| [varmq](https://github.com/goptics/varmq) | v1.10.0 | pool, redis, sqlite | +| [ants](https://github.com/panjf2000/ants) | v2.12.1 | pool | +| [workerpool](https://github.com/gammazero/workerpool) | v1.2.2 | pool | +| [conc](https://github.com/sourcegraph/conc) | v0.3.0 | pool | +| [x/sync](https://github.com/golang/sync) (errgroup) | v0.24.0 | pool | +| [bytedance/gopkg](https://github.com/bytedance/gopkg) (gopool) | v0.1.4 | pool | +| [pond](https://github.com/alitto/pond) | v2.7.2 | pool | +| [redisq](https://github.com/goptics/redisq) | v1.5.0 | redis | +| [Asynq](https://github.com/hibiken/asynq) | v0.26.0 | redis | +| [rmq](https://github.com/adjust/rmq) | v5.2.0 | redis | +| [taskq](https://github.com/vmihailenco/taskq) | v3.2.9 | redis | +| [gocraft/work](https://github.com/gocraft/work) | v0.5.1 | redis | +| [go-redis](https://github.com/redis/go-redis) | v9.23.0 (taskq uses v8.11.5) | redis | +| [redigo](https://github.com/gomodule/redigo) | v1.9.3 | redis | +| [sqliteq](https://github.com/goptics/sqliteq) | v0.3.0 | sqlite | +| [goqite](https://github.com/maragudk/goqite) | v0.4.0 | sqlite | +| [backlite](https://github.com/mikestefanello/backlite) | v0.8.0 | sqlite | +| [River](https://github.com/riverqueue/river) with its [SQLite driver](https://github.com/riverqueue/river/tree/master/riverdriver/riversqlite) | v0.49.0 | sqlite | +| [go-sqlite3](https://github.com/mattn/go-sqlite3) | v1.14.28 | sqlite | ## Running -Needs Go 1.26+ and [Task](https://taskfile.dev); Docker for the Redis suite; a C compiler for the SQLite suite; [vizb](https://github.com/goptics/vizb) and `jq` for `task site:preview`. +Needs Go 1.26+, [Task](https://taskfile.dev), Docker for the Redis suite, a C compiler for the SQLite suite, and [vizb](https://github.com/goptics/vizb) and `jq` for `task site:preview`. + +varmq v1.10.0 is not tagged yet; each go.mod builds it from a checkout of [goptics/varmq](https://github.com/goptics/varmq) at `../../varmq` through a `replace` directive. ```bash task --list # every task diff --git a/pool/README.md b/pool/README.md index a7ca2de..e7726af 100644 --- a/pool/README.md +++ b/pool/README.md @@ -1,32 +1,35 @@ # Worker pool benchmarks -Nine in-process Go worker pools driven by one harness, 1M tasks per op, recorded at 24, 8 and 4 CPUs: [pool-24cpu](https://benchmarks.goptics.org/?id=pool-24cpu), [pool-8cpu](https://benchmarks.goptics.org/?id=pool-8cpu), [pool-4cpu](https://benchmarks.goptics.org/?id=pool-4cpu). +Nine Go worker pools run by one harness, 1M tasks per op: [ants](https://github.com/panjf2000/ants), [bytedance/gopkg](https://github.com/bytedance/gopkg) gopool, [conc](https://github.com/sourcegraph/conc), [pond](https://github.com/alitto/pond), [varmq](https://github.com/goptics/varmq), [workerpool](https://github.com/gammazero/workerpool), [x/sync](https://github.com/golang/sync) errgroup and two channel baselines. +Charts at 24, 8 and 4 CPUs: [pool-24cpu](https://benchmarks.goptics.org/?id=pool-24cpu), [pool-8cpu](https://benchmarks.goptics.org/?id=pool-8cpu), [pool-4cpu](https://benchmarks.goptics.org/?id=pool-4cpu). ## Results -Run of 2026-10-24, varmq 415ad73 (unreleased, `replace` in go.mod), `-benchtime=2x`, 495 rows per file. Each file has 55 task/size/workload rows (11 groups like `Loop10W50K` x 5 workloads); the table counts the rows in which each subject has the lowest `ns/op`, and no other subject is fastest in any row. +Run of 2026-10-24, varmq v1.10.0, `-benchtime=2x`, 495 rows per file. Each file has 55 task/size/workload rows (11 groups such as `Loop10W50K` x 5 workloads), each with 9 subjects. The table counts the rows in which each subject has the lowest `ns/op`; no other subject is fastest in any row. -| CPUs | Rows | VarMQ | BufferedPool | ErrGroup | Ants | -|------|------|-------|--------------|----------|------| -| 24 | 55 | 40 | 13 | 2 | 0 | -| 8 | 55 | 36 | 13 | 6 | 0 | -| 4 | 55 | 20 | 20 | 11 | 4 | +| CPUs | Rows | VarMQ | Ants | ErrGroup | BufferedPool | +|------|------|-------|------|----------|--------------| +| 24 | 55 | 40 | 0 | 2 | 13 | +| 8 | 55 | 36 | 0 | 6 | 13 | +| 4 | 55 | 20 | 4 | 11 | 20 | -On the five one-worker `Loop10W1` rows of each file, VarMQ's `ns/op` divided by BufferedPool's is 3.26x to 4.22x on the four workloads other than `1Mu-1t` (4.22x on `1u-1Mt` at 24 CPUs) and level on `1Mu-1t` (1.04x at 24 CPUs, 0.96x at 8, 1.08x at 4). +On the 15 one-worker `Loop10W1` rows (5 workloads x 3 files), BufferedPool is fastest in 14. On the four workloads other than `1Mu-1t`, the next subject (Gopool on `1u-1Mt`, VarMQ on the other three) takes 2.02x to 3.81x BufferedPool's `ns/op`. On `1Mu-1t`, BufferedPool and VarMQ are the two fastest, and the slower of them takes 1.04x, 1.04x and 1.08x the faster one's `ns/op` at 24, 8 and 4 CPUs. BufferedPool is fastest at 24 and 4 CPUs, VarMQ at 8. ## Subjects -| Name | Version | What it is | -|------|---------|------------| -| VarMQ | 415ad73 (unreleased, `replace` in go.mod) | [goptics/varmq](https://github.com/goptics/varmq), worker bound to a queue | -| Pond | v2.7.2 | [alitto/pond/v2](https://github.com/alitto/pond) | -| Ants | v2.12.1 | [panjf2000/ants/v2](https://github.com/panjf2000/ants) | -| WorkerPool | v1.2.2 | [gammazero/workerpool](https://github.com/gammazero/workerpool) | -| Conc | v0.3.0 | [sourcegraph/conc](https://github.com/sourcegraph/conc) `pool` with a goroutine cap | -| ErrGroup | v0.24.0 | [x/sync/errgroup](https://pkg.go.dev/golang.org/x/sync/errgroup): one goroutine per task behind a semaphore, not a pool | -| Gopool | v0.1.4 | [bytedance/gopkg](https://github.com/bytedance/gopkg) `util/gopool`: submit never blocks; soft worker cap | -| GoroutinePool | baseline | N goroutines reading from an unbuffered channel | -| BufferedPool | baseline | Same, but the channel holds 1M tasks | +| Name | Version | What it is | Submit when the pool is full | Teardown | +|------|---------|------------|------------------------------|----------| +| VarMQ | v1.10.0 | [varmq](https://github.com/goptics/varmq) worker bound to an in-memory queue (`BindQueue`) | Never blocks: `q.Add` appends to an unbounded queue and signals the worker with a non-blocking send | `WaitAndStop`: waits until no job is queued or running, then signals its workers to stop and returns without waiting for them to exit | +| Ants | v2.12.1 | [ants](https://github.com/panjf2000/ants) `NewPool` with default options | `Submit` blocks until a worker is free | `Release`, returns without waiting for its workers to exit | +| WorkerPool | v1.2.2 | [workerpool](https://github.com/gammazero/workerpool) | Never blocks: `Submit` enqueues the task | `StopWait`, waits for its workers | +| Conc | v0.3.0 | [conc](https://github.com/sourcegraph/conc) `pool` with a goroutine cap | `p.Go` blocks at the cap | `Wait`, waits for its goroutines | +| ErrGroup | v0.24.0 | [x/sync](https://github.com/golang/sync) `errgroup` with `SetLimit`: one goroutine per task, not a pool | `g.Go` blocks at the limit | `Wait`, waits for its goroutines | +| Gopool | v0.1.4 | [bytedance/gopkg](https://github.com/bytedance/gopkg) `util/gopool`, soft worker cap | Never blocks: unbounded task list | None; idle workers exit on their own | +| GoroutinePool | baseline | N goroutines reading one unbuffered channel | Blocks: the send waits for a worker to receive | Closes the channel, waits for all N workers to exit | +| BufferedPool | baseline | Same, with a channel that holds 1M tasks | Never blocks: the channel holds all 1M | Closes the channel, waits for all N workers to exit | +| Pond | v2.7.2 | [pond](https://github.com/alitto/pond) v2 `NewPool` | Never blocks: `DefaultQueueSize` is `Unbounded` | `StopAndWait`, waits for its workers | + +Ants, Gopool and VarMQ do not wait for their workers to exit in teardown, so they skip that cost inside the timed region. All subjects live in [subjects.go](subjects.go). @@ -40,12 +43,16 @@ All subjects live in [subjects.go](subjects.go). | `10Ku-100t` | 10,000 | 100 | | `1Mu-1t` | 1,000,000 | 1 | -| Task | Behaviour | -|------|-----------| -| `Sleep10ms` | I/O-like: sleep 10 ms | -| `Loop10` | CPU-like: 10 iterations of trivial math | -| `Sleep5msLoop5` | Mixed: 5 iterations then sleep 5 ms | -| `Loop10K` | CPU-like: 10,000 iterations, about 1 µs. Pool of 1 only. | +A producer is a goroutine. The harness spawns the producers one after another in a loop, with no start barrier. Each producer submits its share of tasks and exits. Producers are not joined; the run waits for task completions. `1Mu-1t` therefore spawns one million goroutines inside the timed region, each submitting one task. + +| Task | What it runs | +|------|--------------| +| `Sleep10ms` | `time.Sleep(10ms)`: a timer park, not I/O | +| `Loop10` | A counted loop of 10 iterations | +| `Sleep5msLoop5` | A 5-iteration loop, then a 5 ms sleep | +| `Loop10K` | A counted loop of 10,000 iterations; pool of one only | + +The loop body is `_ = i * i`, so the compiler removes the multiplication and the CPU tasks are empty counted loops. `Loop10K` takes about 1 µs on this machine: BufferedPool's `Loop10KW1` rows come to 1.02 to 1.18 µs per task. | Size | Tasks | |------|-------| @@ -54,27 +61,29 @@ All subjects live in [subjects.go](subjects.go). | 300K workers | `Sleep10ms`, `Loop10`, `Sleep5msLoop5` | | 1 worker | `Loop10`, `Loop10K` | -W1 runs CPU tasks only, since one worker sleeping 10 ms per task would take about three hours for 1M tasks. +GoroutinePool and BufferedPool start all N workers when the pool is built, so 300K goroutines start inside the timed region at `W300K`. BufferedPool's channel buffer holds all 1M tasks. W1 runs CPU tasks only, since one worker sleeping 10 ms per task would need about 2.8 hours. -The matrix is recorded at 24, 8 and 4 CPUs (`-cpu`), one dataset each. +The matrix is recorded with `-cpu` 24, 8 and 4, which sets GOMAXPROCS without pinning cores; rows carry a `-24`, `-8` or `-4` suffix. ## Method -A subject is a factory that, given `maxWorkers`, returns `submit` and `teardown`. Producers are goroutines: a `100u-10Kt` run starts 100 goroutines, each submitting 10,000 tasks. One op is build the pool, submit 1M tasks, wait for all to finish, tear down, with construction and shutdown inside the timed region. +The timed region of one op covers building the pool, spawning the producers, every submit, the wait until all 1M tasks have completed, and teardown. Completions are counted by the harness's own WaitGroup around each task, not by any pool signal. + +- `ns/op`: wall time of that whole region +- `ops/s`: the same measurement as a rate, 1M x `b.N` / elapsed +- `B/op` and `allocs/op`: allocations of the whole process during the op, including any per-task closure an adapter allocates -- `ns/op`: wall time to push all 1M tasks through the pool and drain it -- `ops/s`: tasks completed per second (custom metric, higher is better) -- `B/op` and `allocs/op`: memory cost +`-benchtime=2x` means two ops per row, with no warm-up op. ## Running | Task | What it does | |------|--------------| -| `task pool:test` | `go vet` and `go test` | -| `task pool:bench` | Every benchmark once (`-benchtime=1x`) | -| `task pool:txt` | Full matrix at 24, 8 and 4 CPUs into `bench/pool-24cpu.txt`, `bench/pool-8cpu.txt` and `bench/pool-4cpu.txt` (hours) | +| `task pool:test` | `go vet ./...` and `go test ./...` | +| `task pool:bench` | Every benchmark once: `-benchtime=1x -benchmem -timeout 0` | +| `task pool:txt` | Full matrix with `-benchtime=2x -benchmem -timeout 0` at `-cpu` 24, 8 and 4 into `bench/pool-24cpu.txt`, `bench/pool-8cpu.txt` and `bench/pool-4cpu.txt`; the committed files took 943.059 s, 724.893 s and 711.582 s | -Rows are named `BenchmarkTasks/W//`, with `W` for workers and sizes in thousands ending in `K`: `BenchmarkTasks/Loop10W50K/100u-10Kt/VarMQ` is the `Loop10` task on 50,000 workers, 100 producers x 10,000 tasks each, subject VarMQ. One slice, the pool of one worker: +Rows are named `BenchmarkTasks/W//-`, with `W` for workers and sizes in thousands ending in `K`. `BenchmarkTasks/Loop10W50K/100u-10Kt/VarMQ-24` is the `Loop10` task on 50,000 workers, 100 producers x 10,000 tasks each, subject VarMQ, at `-cpu 24`. One slice, the pool of one worker: ```bash go test -run xxx -bench 'Tasks/W1$' -benchtime=1x diff --git a/redis/NOTES.md b/redis/NOTES.md index 85122c2..feabc3e 100644 --- a/redis/NOTES.md +++ b/redis/NOTES.md @@ -15,7 +15,7 @@ Per-library notes read from the library sources at the versions in go.mod; the t ### VarMQ (persistent) -Library versions `varmq 415ad73` (unreleased, `replace` directive) and +Library versions `varmq v1.10.0` and `redisq v1.5.0`. - Dequeue: `bind` (`varmq/source.go`) sees that `*redisq.Queue` implements @@ -207,7 +207,7 @@ Library version `asynq v0.26.0`. Paths are inside the module (`go env GOMODCACHE ### VarMQDistributed -Library versions `varmq 415ad73` (unreleased, `replace` directive) and +Library versions `varmq v1.10.0` and `redisq v1.5.0`. - Dequeue and ack: the same credit path as VarMQ (persistent). diff --git a/redis/README.md b/redis/README.md index 4e36afc..51195f9 100644 --- a/redis/README.md +++ b/redis/README.md @@ -1,13 +1,27 @@ -# Redis message queue benchmarks +# Redis job queue benchmarks -VarMQ (redisq) against Asynq, rmq, taskq and gocraft/work on Redis. +Six subjects from five Go job-queue libraries backed by Redis: [Asynq](https://github.com/hibiken/asynq), [gocraft/work](https://github.com/gocraft/work), [rmq](https://github.com/adjust/rmq), [taskq](https://github.com/vmihailenco/taskq) and VarMQ ([varmq](https://github.com/goptics/varmq) with [redisq](https://github.com/goptics/redisq): a single-instance persistent queue and a distributed queue). Chart: https://benchmarks.goptics.org/?id=redis +## Workloads + +- One op of `BenchmarkDurable` moves 100,000 jobs through freshly built clients. The timed region covers building every consumer instance and, where it is separate, the producer client; the Redis connection dials; every enqueue; draining all 100,000 jobs; and stopping everything. The `FLUSHDB` before each iteration is not timed. Nothing is reused between iterations. +- Payload: JSON `{"id": N, "t": , "pad": <48 zero bytes as a JSON array>}`, 136 to 140 bytes. Each library wraps it in its own envelope (see [NOTES.md](NOTES.md)). +- `1u-100Kt` is 1 producer goroutine enqueuing 100,000 jobs one after another. `100u-1Kt` is 100 producer goroutines enqueuing 1,000 each, concurrently. All producers share one producer client. For `VarMQ` the producer is consumer instance 0's own queue, because its persistent queue wakes its worker in process. +- `W` is the workers per instance, passed as each library's concurrency setting. +- `C` is the number of library instances inside the same `go test` process, each with its own client and Redis connection pool. `C4` means four instances and 4 x W workers. They are not separate OS processes. +- `Noop` runs nothing in the handler beyond the harness's duplicate check. `Sleep1ms` sleeps 1 ms, so a row is capped at C x W x 1,000 jobs/s before setup and teardown are counted. +- `VarMQ` runs `C1` only, because its persistent queue is single-instance. `Asynq`, `GocraftWork`, `Rmq`, `Taskq` and `VarMQDistributed` run `C1` and `C4`. On the chart the `C1`, Enqueue and Latency groups leave `VarMQDistributed` out; its rows for them are in `bench/redis.txt`. +- `BenchmarkEnqueue` runs the producer only, with no consumers, on the same two workloads and payload. It is timed from building the producer to its close. +- `BenchmarkLatency` measures enqueue-to-handler latency: 1,000 jobs, one at a time, on one instance with 10 workers. Job i+1 is enqueued only after job i reached the handler, with a 10 s limit per job. Each sample is the time from just before the enqueue call to handler entry, so it includes the enqueue round trip. p50, p99 and max are over all samples of the row's iterations: 2,000 samples at `-benchtime=2x`. +- A `Durable` row fails if the 100,000 unique jobs are not all handled within 5 minutes or an enqueue fails. Duplicate deliveries are counted but do not fail a row; `TestSubjects` (`task redis:test`) fails on them. +- `jobs/s` = jobs x iterations / timed seconds. + ## Results -Run of 2026-10-25: varmq 415ad73 (unreleased, `replace` in go.mod), redisq v1.5.0, `-benchtime=2x`, Redis 7.2 (Debian image) pinned to one core, 150 rows in `bench/redis.txt`. +Run of 2026-10-25: varmq v1.10.0, redisq v1.5.0, `-benchtime=2x`, Redis 7.2 pinned to one core, 150 rows in `bench/redis.txt`. -`C1`: jobs/s, `Noop` task, one consumer process. +`C1`: jobs/s, `Noop` task, one library instance. | Row | VarMQ | Asynq | Rmq | Taskq | GocraftWork | |---|---:|---:|---:|---:|---:| @@ -18,25 +32,29 @@ Run of 2026-10-25: varmq 415ad73 (unreleased, `replace` in go.mod), redisq v1.5. | NoopW1000C1 / 1u-100Kt | 19,710 | 16,467 | 32,678 | 16,674 | 24,408 | | NoopW1000C1 / 100u-1Kt | 58,523 | 17,932 | 27,349 | 28,638 | 40,679 | -`C4`: jobs/s, `Noop` task, four consumer processes. +Rmq leads every single-producer row; at 100 producers GocraftWork leads at W10 and VarMQ at W100 and W1000. -| Row | VarMQDistributed | Asynq | Rmq | Taskq | GocraftWork | +`C4`: jobs/s, `Noop` task, four library instances. + +| Row | Asynq | VarMQDistributed | Rmq | Taskq | GocraftWork | |---|---:|---:|---:|---:|---:| -| NoopW10C4 / 1u-100Kt | 22,057 | 11,896 | 29,616 | 10,904 | 24,339 | -| NoopW10C4 / 100u-1Kt | 53,754 | 20,889 | 59,649 | 17,323 | 48,722 | -| NoopW100C4 / 1u-100Kt | 22,263 | 13,350 | 29,953 | 10,576 | 23,683 | -| NoopW100C4 / 100u-1Kt | 63,710 | 22,677 | 58,620 | 16,077 | 41,730 | -| NoopW1000C4 / 1u-100Kt | 22,231 | 12,747 | 27,784 | 8,267 | 21,068 | -| NoopW1000C4 / 100u-1Kt | 64,281 | 19,119 | 52,549 | 11,100 | 36,809 | +| NoopW10C4 / 1u-100Kt | 11,896 | 22,057 | 29,616 | 10,904 | 24,339 | +| NoopW10C4 / 100u-1Kt | 20,889 | 53,754 | 59,649 | 17,323 | 48,722 | +| NoopW100C4 / 1u-100Kt | 13,350 | 22,263 | 29,953 | 10,576 | 23,683 | +| NoopW100C4 / 100u-1Kt | 22,677 | 63,710 | 58,620 | 16,077 | 41,730 | +| NoopW1000C4 / 1u-100Kt | 12,747 | 22,231 | 27,784 | 8,267 | 21,068 | +| NoopW1000C4 / 100u-1Kt | 19,119 | 64,281 | 52,549 | 11,100 | 36,809 | + +Rmq leads every single-producer row; at 100 producers Rmq leads at W10 and VarMQDistributed at W100 and W1000. -`Sleep1ms` rows are bound by workers x 1 ms and are only on the chart. +`Sleep1ms` rows are capped at C x W x 1,000 jobs/s and are on the chart and in `bench/redis.txt`. -Enqueue only (`BenchmarkEnqueue`, jobs/s): at one producer VarMQ 37,292, Rmq 33,394, GocraftWork 29,720, VarMQDistributed 27,183, Taskq 26,780, Asynq 24,023; at 100 producers VarMQ 207,752, GocraftWork 202,835, Rmq 197,240, Taskq 188,437, VarMQDistributed 144,838, Asynq 121,046. +Enqueue only (`BenchmarkEnqueue`, jobs/s, highest first): at one producer VarMQ 37,292, Rmq 33,394, GocraftWork 29,720, VarMQDistributed 27,183, Taskq 26,780, Asynq 24,023; at 100 producers VarMQ 207,752, GocraftWork 202,835, Rmq 197,240, Taskq 188,437, VarMQDistributed 144,838, Asynq 121,046. -Pickup latency (`BenchmarkLatency`, 1,000 jobs one at a time, ms): +Enqueue-to-handler latency (`BenchmarkLatency`, ms, lowest p50 first): | Subject | p50 | p99 | max | -|---|---|---|---| +|---|---:|---:|---:| | Taskq | 0.03876 | 0.1623 | 3.195 | | VarMQDistributed | 0.06492 | 0.2351 | 2.436 | | VarMQ | 0.09407 | 0.5321 | 3.431 | @@ -44,39 +62,41 @@ Pickup latency (`BenchmarkLatency`, 1,000 jobs one at a time, ms): | Rmq | 1.464 | 2.891 | 6.902 | | Asynq | 979.7 | 1491 | 1501 | -Asynq's 0.5 to 1.5 s idle sleep shows directly, and rmq's 1 ms poll puts it at 1.464 ms. Taskq's blocking `XREADGROUP` is the fastest pickup at 0.03876 ms; VarMQ's batch pop is at 0.09407 ms and VarMQDistributed's push wake at 0.06492 ms. +Asynq sleeps 0.5 to 1.5 s after an empty poll, and its numbers are that sleep. +Rmq sleeps 1 ms after every fetch pass, which puts its p50 at 1.464 ms. +Taskq's fetcher blocks in `XREADGROUP`, so the producer's `XADD` wakes it directly. +VarMQ's `Add` wakes its worker in the same process, which then pops the job with one script. +VarMQDistributed's enqueue script publishes a wake to one idle consumer. -In `C4` rows the four instances are stopped one after another inside the timed region, so Asynq and Taskq pay their idle poll up to four times. +Every row's setup and teardown are timed. In `C4` rows the four instances are stopped one after another, so Asynq and Taskq pay their idle poll up to four times. Full run: `bench/redis.txt`. -On the chart, the C1, Enqueue and Latency groups show VarMQ and the four rivals; the C4 groups show VarMQDistributed and the four. ## Subjects | Subject (chart name) | Library | Version | Instances | Wake | Consumer round trips per job | |---|---|---|---|---|---| -| VarMQ | goptics/varmq with goptics/redisq | varmq 415ad73 (unreleased, `replace` in go.mod), redisq v1.5.0 | 1 | push in-process | 1 ack pipeline per job + 1 pop script per up to 64 jobs | -| VarMQDistributed | goptics/varmq with goptics/redisq | varmq 415ad73 (unreleased, `replace` in go.mod), redisq v1.5.0 | 1 and 4 | push to one idle consumer plus a 1 s nudge | 1 ack pipeline per job + 1 pop script per up to 64 jobs | -| Asynq | hibiken/asynq | v0.26.0 | 1 and 4 | poll 0.5-1.5 s | 2 | -| Rmq | adjust/rmq/v5 | v5.2.0 | 1 and 4 | poll 1 ms | 2 | -| Taskq | vmihailenco/taskq/v3 | v3.2.9 with go-redis v8 v8.11.5 | 1 and 4 | blocking XREADGROUP | 2 + 1 per 10 jobs | -| GocraftWork | gocraft/work | v0.5.1 with redigo v1.9.3 | 1 and 4 | backoff poll 10 ms to 5 s | 2 | +| VarMQ | [goptics/varmq](https://github.com/goptics/varmq) with [goptics/redisq](https://github.com/goptics/redisq) | varmq v1.10.0, redisq v1.5.0, [go-redis](https://github.com/redis/go-redis) v9.23.0 | 1 | push in-process | 1 ack pipeline per job + 1 pop script per up to 64 jobs | +| VarMQDistributed | [goptics/varmq](https://github.com/goptics/varmq) with [goptics/redisq](https://github.com/goptics/redisq) | varmq v1.10.0, redisq v1.5.0, [go-redis](https://github.com/redis/go-redis) v9.23.0 | 1 and 4 | push to one idle consumer plus a 1 s nudge | 1 ack pipeline per job + 1 pop script per up to 64 jobs | +| Asynq | [hibiken/asynq](https://github.com/hibiken/asynq) | v0.26.0 with [go-redis](https://github.com/redis/go-redis) v9.23.0 | 1 and 4 | poll 0.5-1.5 s | 2 | +| Rmq | [adjust/rmq](https://github.com/adjust/rmq) | v5.2.0 with [go-redis](https://github.com/redis/go-redis) v9.23.0 | 1 and 4 | poll 1 ms | 2 | +| Taskq | [vmihailenco/taskq](https://github.com/vmihailenco/taskq) | v3.2.9 with [go-redis](https://github.com/redis/go-redis) v8.11.5 | 1 and 4 | blocking XREADGROUP | 2 + 1 per 10 jobs | +| GocraftWork | [gocraft/work](https://github.com/gocraft/work) | v0.5.1 with [redigo](https://github.com/gomodule/redigo) v1.9.3 | 1 and 4 | backoff poll 10 ms to 5 s | 2 | What each library does per job, read from source: [NOTES.md](NOTES.md). -## Matrix +## Settings -| Benchmark | Varies | Metric | -|---|---|---| -| Durable | tasks `Noop`, `Sleep1ms`; workers 10, 100, 1000; instances 1, 4; workloads `1u-100Kt` (1 producer x 100,000 jobs), `100u-1Kt` (100 x 1,000) | jobs/s | -| Enqueue | workloads `1u-100Kt`, `100u-1Kt` | jobs/s | -| Latency | `1u-1Kt`, W10 C1 | p50-ms, p99-ms, max-ms | +Each subject runs at its library defaults except: -Names: `BenchmarkDurable/WC//`, `BenchmarkEnqueue/Enqueue//`, `BenchmarkLatency/LatencyW10C1/1u-1Kt/`. - -Each subject runs at its library defaults except worker concurrency and the Redis pool size, `max(workers, 10*GOMAXPROCS)`; the settings a library forces are listed in [NOTES.md](NOTES.md). rmq has no default for its poll duration or prefetch limit; the harness uses 1 ms and `max(1000, 2*workers)` (the example's 100 ms capped a connection near prefetch / 0.1 s), see [NOTES.md](NOTES.md). +- Worker concurrency is W. +- The Redis pool size is `max(workers, 10 x GOMAXPROCS)` per consumer instance: 220 for W10 and W100 here (GOMAXPROCS 22), 1,000 for W1000. Every producer client gets 10 x GOMAXPROCS (220), go-redis's default. +- rmq polls every 1 ms with a prefetch limit of `max(1000, 2 x workers)`, because rmq has no default for either. The 100 ms of rmq's example capped a connection near prefetch / 0.1 s. +- taskq runs with `MinNumWorker = MaxNumWorker = workers`, which turns off its autotuning. +- gocraft/work's redigo pool has `Wait: true`, so a full pool blocks instead of failing. +- Asynq uses one queue, named `bench`. -VarMQ's persistent queue runs one instance, so it has no `C4` rows. +Names: `BenchmarkDurable/WC//`, `BenchmarkEnqueue/Enqueue//`, `BenchmarkLatency/LatencyW10C1/1u-1Kt/`. ## Running diff --git a/sqlite/README.md b/sqlite/README.md index 914ed4d..a9217c7 100644 --- a/sqlite/README.md +++ b/sqlite/README.md @@ -1,11 +1,13 @@ -# SQLite message queue benchmarks +# SQLite job queue benchmarks -VarMQ with [sqliteq](https://github.com/goptics/sqliteq) against [goqite](https://github.com/maragudk/goqite), [backlite](https://github.com/mikestefanello/backlite) and [River](https://github.com/riverqueue/river)'s SQLite driver, end to end in one process. +Five subjects from four Go job-queue libraries backed by SQLite: [backlite](https://github.com/mikestefanello/backlite), [goqite](https://github.com/maragudk/goqite), [River](https://github.com/riverqueue/river) through its SQLite driver [riversqlite](https://github.com/riverqueue/river/tree/master/riverdriver/riversqlite), and [varmq](https://github.com/goptics/varmq) with [sqliteq](https://github.com/goptics/sqliteq), run as two subjects: VarMQ, with one database file per queue, and VarMQSingleFile, with every queue in one file. +Every subject runs end to end in one process, and all of them reach SQLite through [go-sqlite3](https://github.com/mattn/go-sqlite3). Chart: https://benchmarks.goptics.org/?id=sqlite ## Results -Run of 2026-10-25: varmq 415ad73 (unreleased, `replace` in go.mod), sqliteq v0.3.0, goqite v0.4.0, backlite v0.8.0, River v0.49.0, `-benchtime=2x`, 32 rows in `bench/sqlite.txt`, ext4 over NVMe. jobs/s, higher is better; walMB is the peak total size of the run's WAL files, lower is better. +Run of 2026-10-25: backlite v0.8.0, go-sqlite3 v1.14.28, goqite v0.4.0, River v0.49.0, sqliteq v0.3.0, varmq v1.10.0, `-benchtime=2x`, 32 rows in `bench/sqlite.txt`, ext4 over NVMe. +Higher jobs/s is better; lower walMB (the peak total size of the run's WAL files) is better. | Scenario | Subject | ckpt1000 | ckpt10000 | walMB 1000 / 10000 | |----------|---------|---------:|----------:|-------------------:| @@ -26,37 +28,53 @@ Run of 2026-10-25: varmq 415ad73 (unreleased, `replace` in go.mod), sqliteq v0.3 | `Mixed10Q` | Backlite | 4,807 | 5,781 | 352.9 / 369.6 | | `Mixed10Q` | River | 1,383 | 2,173 | 4.3 / 41.2 | -VarMQ is first in all 8 scenario and checkpoint groups; on `Mixed1Q` at ckpt1000 it runs 3.1x goqite, 4.1x River and 23x backlite. A 10,000-page checkpoint is 1.2 to 3.7x faster than 1,000 for VarMQ, goqite and River and holds a WAL 9 to 10 times larger (4.2 MB against 41.2 MB per file). Ten queues in separate files (VarMQ) run 6.1x and 9.9x the same ten in one file (VarMQSingleFile), with ten WAL files instead of one: 41.3 and 385.4 MB in total against 4.2 and 41.3. Backlite's WAL peaks at 114 to 370 MB whatever the checkpoint size and on `Drain1Q` it is slower at the larger one. +VarMQ is the fastest subject in all 8 scenario and checkpoint groups. +On `Mixed1Q` the rest follow in the order Goqite, River, Backlite at both checkpoint sizes. +On `Drain1Q` and `DrainPrio`, River is ahead of Goqite at ckpt1000 and Goqite is ahead of River at ckpt10000; Backlite is last on `Drain1Q` at both. +On `Mixed10Q` the order after VarMQ is Backlite, VarMQSingleFile, Goqite, River at ckpt1000 and VarMQSingleFile, Backlite, Goqite, River at ckpt10000. +At ckpt10000, jobs/s is 1.2 to 3.7 times the ckpt1000 figure for Goqite, River, VarMQ and VarMQSingleFile in every scenario. +For Backlite the ratio is 3.5 on `Mixed1Q` and 1.2 on `Mixed10Q`; on `Drain1Q` its jobs/s falls from 1,286 to 1,168. +In every row with one database file except Backlite's, walMB is 4.1 to 4.7 MB at ckpt1000 and 41.2 to 41.3 MB at ckpt10000. +Backlite's WAL peaks at 113.8 to 352.9 MB at ckpt1000 and 166.9 to 369.6 MB at ckpt10000. +On `Mixed10Q`, ten queues in ten files (VarMQ) reach 6.1 times the jobs/s of the same ten queues in one file (VarMQSingleFile) at ckpt1000 and 9.9 times at ckpt10000. +The ten WAL files total 41.3 and 385.4 MB, against 4.2 and 41.3 MB for the single file. Absolute numbers vary between runs on the same machine; compare ratios within one run. ## Subjects -| Subject | Library | Version | SQLite connections | Consumer | -|---------|---------|---------|--------------------|----------| -| VarMQ | sqliteq through a varmq worker | sqliteq v0.3.0, varmq 415ad73 | pool of 8 per file (sqliteq) | one database file per queue (sqliteq's default), 16 concurrent jobs per queue | -| VarMQSingleFile | same | same | same | every queue in one file (`WithSingleFile`); `Mixed10Q` only | -| Goqite | maragu.dev/goqite with its `jobs` runner | v0.4.0 | one (its README example) | one runner per queue, limit 16, 100 ms poll between empty receives | -| Backlite | mikestefanello/backlite | v0.8.0 | default pool, unbounded | one client with 16 x queues workers shared by every queue, dispatcher woken in-process on every add | -| River | riverqueue/river with riverdriver/riversqlite | v0.49.0 | one (its driver doc) | one client, 16 workers per queue, 1 ms fetch cooldown per queue, completions written in batches by its completer | - -Every subject opens SQLite with WAL, synchronous NORMAL, a 5 s busy timeout and the row's `wal_autocheckpoint`. Everything else is the library default, apart from three settings: River's per-queue `FetchCooldown` is its 1 ms minimum, since a queue's producer fetches at most `MaxWorkers` jobs per cooldown and the 100 ms default caps 16 workers at 160 jobs/s whatever the database does (the client-level value, which also paces River's insert notifications, stays at the default); River's logger is discarded; backlite's `ReleaseAfter`, which has no default, is 10 minutes. +| Subject | Library | Version | Database files | SQLite connections | Stored payload | Consumer | +|---------|---------|---------|----------------|--------------------|----------------|----------| +| VarMQ | [varmq](https://github.com/goptics/varmq) with [sqliteq](https://github.com/goptics/sqliteq) | varmq v1.10.0, sqliteq v0.3.0 | one per queue (sqliteq's default) | pool of 8 per file | varmq's job JSON (`id`, `status`, `data`) | one varmq worker per queue, 16 concurrent jobs | +| VarMQSingleFile | [varmq](https://github.com/goptics/varmq) with [sqliteq](https://github.com/goptics/sqliteq) | varmq v1.10.0, sqliteq v0.3.0 | one `bench.db` for every queue (`WithSingleFile`) | pool of 8 | as VarMQ | as VarMQ; `Mixed10Q` only | +| Goqite | [goqite](https://github.com/maragudk/goqite) with its `jobs` runner | v0.4.0 | one `bench.db` | one (its README's setup) | the int's decimal bytes inside goqite's gob job envelope | one runner per queue, limit 16; goqite's default 100 ms poll between empty receives and while all 16 slots are busy | +| Backlite | [backlite](https://github.com/mikestefanello/backlite) | v0.8.0 | one `bench.db` | default pool, unbounded | JSON `{"N": i}` | one client with 16 x queues workers shared by every queue; dispatcher woken in process on every add | +| River | [River](https://github.com/riverqueue/river) with [riversqlite](https://github.com/riverqueue/river/tree/master/riverdriver/riversqlite) | v0.49.0 | one `bench.db` | one (its driver doc) | JSON `{"n": i}` | one client, 16 workers per queue, per-queue `FetchCooldown` 1 ms; completions written in batches by its completer | + +Every subject opens SQLite with WAL, synchronous NORMAL, a 5 s busy timeout and the row's `wal_autocheckpoint`, through the same DSN and connect hook as sqliteq. +Beyond the table, everything is the library default except three settings. +Backlite's `ReleaseAfter` is 10 minutes, since it has no default. +River's per-queue `FetchCooldown` is 1 ms: a queue fetches at most `MaxWorkers` jobs per cooldown, so the 100 ms default caps 16 workers at 160 jobs/s whatever the database does. +River's client-level `FetchCooldown`, which also paces its insert notifications, stays at its default. +River's logger is discarded. ## Scenarios -Every job is an `int`; the handler does nothing. - -| Name | Queues | Producers per queue | Jobs | Timed | Subjects | -|------|-------:|--------------------:|-----:|-------|----------| -| `Mixed1Q` | 1 | 4 | 20,000 | first add to last job acknowledged, producing and consuming at once | all | -| `Drain1Q` | 1 | 4 | 20,000 | everything added first, then consumer start (River's client start-up included) to last job acknowledged | all | -| `DrainPrio` | 1 | 4 | 10,000 | as `Drain1Q` on a priority queue, priority `i % 4` | VarMQ, Goqite, River | -| `Mixed10Q` | 10 | 1 | 20,000 | as `Mixed1Q` over ten queues, 16 workers each | all, plus VarMQSingleFile | - -- Rows are named `BenchmarkSQLite//ckpt/`. -- `ckpt` is the WAL checkpoint size in pages, 1000 (SQLite's default) or 10000, set on every connection. -- `jobs/s` is the job count divided by the timed part; setup, teardown and the checks below are untimed. -- `walMB` is the largest total size the `-wal` files in the run's directory reached, sampled every 50 ms from before the store opens, untimed phases included. -- A run fails if any add failed, the handler did not run exactly once per job, or rows remain in the queue tables. +| Name | Queues | Producers per queue | Jobs (total) | Per queue | Timed | Subjects | +|------|-------:|--------------------:|-------------:|-----------|-------|----------| +| `Mixed1Q` | 1 | 4 | 20,000 | 20,000 (5,000 per producer) | mixed | all but VarMQSingleFile | +| `Drain1Q` | 1 | 4 | 20,000 | 20,000 (5,000 per producer) | drain | all but VarMQSingleFile | +| `DrainPrio` | 1 | 4 | 10,000 | 10,000 (2,500 per producer) | drain | VarMQ, Goqite, River | +| `Mixed10Q` | 10 | 1 | 20,000 | 2,000 | mixed | all five | + +- Mixed: the consumers start before the timer, which runs from the first add to the last job acknowledged, producing and consuming at once. +- Drain: every job is added first, then the timer runs from consumer start to the last job acknowledged, so each subject's start-up is timed (backlite `Start`, goqite's runner goroutines, River `client.Start`, varmq `Worker.Start`). +- Every job is an `int`; the handler only increments a shared counter. +- Priority (`DrainPrio`): Goqite, River and VarMQ run it, and each maps `i % 4 == 0` to its highest priority: goqite takes higher numbers first (`3 - i % 4`), River takes 1 as highest (`i % 4 + 1`), varmq takes lower numbers first (`i % 4`). The order in which jobs are served is not checked. +- Rows are named `BenchmarkSQLite//ckpt/`; `ckpt` is the WAL checkpoint size in pages, 1000 (SQLite's default) or 10000, set on every connection. +- jobs/s = jobs x iterations / timed seconds. +- walMB = the largest total size of all `-wal` files in the run's directory, sampled every 50 ms from before the store opens until just before it closes, the maximum over the row's iterations. SQLite does not shrink a WAL file after a checkpoint, so this is the files' high-water size. +- A row fails on an open or schema error, a failed add, a start error, the jobs not all acknowledged within 5 minutes, a handler count different from the job count, rows left in the queue tables (River counts jobs not in state `completed`, since it keeps completed rows), or a close error. +- The handler count is checked as a total, so one lost job plus one duplicate would pass. ## Running @@ -65,7 +83,7 @@ Needs Go 1.26 and a C compiler. ```sh task sqlite:test # vet, then every subject through a small job set task sqlite:bench # every scenario and subject once -task sqlite:txt # full run into bench/sqlite.txt (about 20 min) +task sqlite:txt # full run into bench/sqlite.txt ``` One slice: @@ -74,4 +92,5 @@ One slice: go test -run xxx -bench 'SQLite/Mixed10Q' -benchtime=1x ``` -`SQLITEQ_BENCH_DIR` is where the databases are created (default `/varmq-bench-sqlite`). Use a real disk: durable writes pay for fsync, and `/tmp` is often a tmpfs where that cost vanishes. +`SQLITEQ_BENCH_DIR` is where the databases are created (default `/varmq-bench-sqlite`). +Use a real disk: durable writes pay for fsync, and `/tmp` is often a tmpfs where that cost vanishes.