From 15ea57de30c80e6547a3913dc3ce914b447b9bdf Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 11:18:18 +0600 Subject: [PATCH 1/3] perf: batch pops for varmq and one write mutex per file DequeueBatchWithAckIds claims up to min(len(items), len(ackIDs)) rows in one statement and reports the pending count after it, capped at 256. Every write on a file (enqueue, claim, pop, ack, purge, recovery, schema setup) holds the file's mutex, so writers on one file queue in Go instead of sleeping in SQLite's busy handler. Len and Values do not take it. --- priority_queue_test.go | 30 +++++ queue.go | 100 +++++++++++--- queue_test.go | 287 +++++++++++++++++++++++++++++++++++++++++ registry.go | 2 +- 4 files changed, 401 insertions(+), 18 deletions(-) diff --git a/priority_queue_test.go b/priority_queue_test.go index a80c8fb..39b2ed7 100644 --- a/priority_queue_test.go +++ b/priority_queue_test.go @@ -623,3 +623,33 @@ func TestPriorityOrderByID(t *testing.T) { } } } + +func TestBatchPriorityOrder(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("q") + if err != nil { + t.Fatalf("Failed to create priority queue: %v", err) + } + + // Job names are "/" + for i, p := range []int{3, 1, 1, 0, 2} { + if !pq.Enqueue([]byte(fmt.Sprintf("%d/%d", p, i)), p) { + t.Fatalf("Enqueue %d failed", i) + } + } + + items, ackIDs := make([]any, 5), make([]string, 5) + n, remaining, err := pq.DequeueBatchWithAckIds(items, ackIDs) + if err != nil || n != 5 || remaining != 0 { + t.Fatalf("n=%d remaining=%d err=%v, want 5, 0, nil", n, remaining, err) + } + for i, want := range []string{"0/3", "1/1", "1/2", "2/4", "3/0"} { + if got := string(items[i].([]byte)); got != want { + t.Errorf("items[%d] = %q, want %q", i, got, want) + } + if !pq.Acknowledge(ackIDs[i]) { + t.Errorf("Acknowledge(%s) failed", ackIDs[i]) + } + } +} diff --git a/queue.go b/queue.go index 31fbd20..90ac89c 100644 --- a/queue.go +++ b/queue.go @@ -58,11 +58,8 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro } // The db is shared by every queue of the file, so it is not closed on errors - if err := q.initTable(); err != nil { - return nil, fmt.Errorf("failed to initialize table: %w", err) - } - if err := q.migrate(); err != nil { - return nil, fmt.Errorf("failed to migrate indexes: %w", err) + if err := q.initSchema(); err != nil { + return nil, err } if err := q.prepare(); err != nil { return nil, fmt.Errorf("failed to prepare statements: %w", err) @@ -71,6 +68,20 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro return q, nil } +// initSchema creates or migrates the table and its index under the write mutex +func (q *Queue) initSchema() error { + q.f.mu.Lock() + defer q.f.mu.Unlock() + + if err := q.initTable(); err != nil { + return fmt.Errorf("failed to initialize table: %w", err) + } + if err := q.migrate(); err != nil { + return fmt.Errorf("failed to migrate indexes: %w", err) + } + return nil +} + // initTable creates the table as v0.2 defines it, so old tables need no // rebuild, and adds the priority column to a priority queue's table when // it is missing @@ -200,6 +211,9 @@ func (q *Queue) closeStmts() { // requeueNoAck puts rows left in processing without an ack back to pending func (q *Queue) requeueNoAck() error { + q.f.mu.Lock() + defer q.f.mu.Unlock() + _, err := q.f.db.Exec( fmt.Sprintf("UPDATE %s SET status = 'pending', updated_at = ? WHERE status = 'processing' AND ack = 0", q.table), time.Now().UTC(), @@ -258,6 +272,9 @@ func (q *Queue) enqueue(item any, priority int) bool { } now := time.Now().UTC() + q.f.mu.Lock() + defer q.f.mu.Unlock() + var err error if q.priority { _, err = q.insert.Exec(item, now, now, priority) @@ -270,6 +287,26 @@ func (q *Queue) enqueue(item any, priority int) bool { // claimN moves up to n pending rows to processing in one statement and // returns them in queue order func (q *Queue) claimN(n int) ([]claimed, error) { + q.f.mu.Lock() + out, err := q.claimRows(n) + q.f.mu.Unlock() + if err != nil { + return nil, err + } + + // RETURNING order is unspecified + slices.SortFunc(out, func(a, b claimed) int { + if c := cmp.Compare(a.priority, b.priority); c != 0 { + return c + } + return cmp.Compare(a.id, b.id) + }) + return out, nil +} + +// claimRows runs the claim statement and reads every row it returns; the +// caller holds the write mutex +func (q *Queue) claimRows(n int) ([]claimed, error) { rows, err := q.claim.Query(time.Now().UTC(), n) if err != nil { return nil, err @@ -290,18 +327,7 @@ func (q *Queue) claimN(n int) ([]claimed, error) { out = append(out, c) } // Reading to the end runs the statement to completion, which commits it - if err := rows.Err(); err != nil { - return nil, err - } - - // RETURNING order is unspecified - slices.SortFunc(out, func(a, b claimed) int { - if c := cmp.Compare(a.priority, b.priority); c != 0 { - return c - } - return cmp.Compare(a.id, b.id) - }) - return out, nil + return out, rows.Err() } // Dequeue removes and returns the next item from the queue @@ -311,6 +337,10 @@ func (q *Queue) Dequeue() (any, bool) { return nil, false } + q.f.mu.Lock() + defer q.f.mu.Unlock() + + // Deferred last, so the rows close (and the statement ends) before unlocking rows, err := q.pop.Query() if err != nil { return nil, false @@ -345,6 +375,36 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) { return c[0].data, true, strconv.FormatInt(c[0].id, 10) } +// DequeueBatchWithAckIds implements varmq.IBatchAcknowledgeable: it moves up +// to min(len(items), len(ackIDs)) items to processing in one statement, in +// queue order, each under its row id as ack id. remaining is the pending +// count after the move, capped at remainingCap. n is 0 with a nil error when +// the queue is empty or closed. +func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaining int, err error) { + want := min(len(items), len(ackIDs)) + if q.closed.Load() || want == 0 { + return 0, 0, nil + } + + c, err := q.claimN(want) + if err != nil { + return 0, 0, err + } + if len(c) == 0 { + return 0, 0, nil + } + for i, row := range c { + items[i], ackIDs[i] = row.data, strconv.FormatInt(row.id, 10) + } + + // The rows are claimed already: an error here would make varmq drop them, + // so a failed count reports items left and the next pop finds out + if err := q.countCapped.QueryRow().Scan(&remaining); err != nil { + return len(c), 1, nil + } + return len(c), remaining, nil +} + // Acknowledge marks a processing item as completed, deleting it unless // WithRemoveOnComplete(false) is set // Returns true only if exactly one processing row had this ack ID @@ -354,6 +414,9 @@ func (q *Queue) Acknowledge(ackID string) bool { return false } + q.f.mu.Lock() + defer q.f.mu.Unlock() + var ( result sql.Result err error @@ -405,6 +468,9 @@ func (q *Queue) Values() []any { // Purge removes all items from the queue func (q *Queue) Purge() { + q.f.mu.Lock() + defer q.f.mu.Unlock() + q.f.db.Exec(fmt.Sprintf("DELETE FROM %s", q.table)) } diff --git a/queue_test.go b/queue_test.go index 023bbdd..fb081e8 100644 --- a/queue_test.go +++ b/queue_test.go @@ -8,6 +8,7 @@ import ( "sync" "sync/atomic" "testing" + "time" ) func TestSQLiteQueue(t *testing.T) { @@ -594,3 +595,289 @@ func TestConcurrentClaimsNoDuplicates(t *testing.T) { t.Errorf("Expected no rows left, got %d", n) } } + +func TestImplementsBatch(t *testing.T) { + type batcher interface { + DequeueBatchWithAckIds([]any, []string) (int, int, error) + } + var _ batcher = (*Queue)(nil) + var _ batcher = (*PriorityQueue)(nil) +} + +// enqueueN adds jobs "0".."n-1" to q. +func enqueueN(t *testing.T, q *Queue, n int) { + t.Helper() + for i := 0; i < n; i++ { + if !q.Enqueue([]byte(strconv.Itoa(i))) { + t.Fatalf("Enqueue %d failed", i) + } + } +} + +func TestBatchPop(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 10) + + items, ackIDs := make([]any, 4), make([]string, 4) + n, remaining, err := q.DequeueBatchWithAckIds(items, ackIDs) + if err != nil || n != 4 || remaining != 6 { + t.Fatalf("Batch of 4: n=%d remaining=%d err=%v, want 4, 6, nil", n, remaining, err) + } + seen := map[string]bool{} + for i := 0; i < n; i++ { + if got := string(items[i].([]byte)); got != strconv.Itoa(i) { + t.Errorf("items[%d] = %q, want %q", i, got, strconv.Itoa(i)) + } + if _, ok := parseAckID(ackIDs[i]); !ok { + t.Errorf("ackIDs[%d] = %q is not a row id", i, ackIDs[i]) + } + if seen[ackIDs[i]] { + t.Errorf("ackIDs[%d] = %q repeats", i, ackIDs[i]) + } + seen[ackIDs[i]] = true + } + + items, ackIDs = make([]any, 10), make([]string, 10) + n, remaining, err = q.DequeueBatchWithAckIds(items, ackIDs) + if err != nil || n != 6 || remaining != 0 { + t.Fatalf("Batch of 10: n=%d remaining=%d err=%v, want 6, 0, nil", n, remaining, err) + } + for i := 0; i < n; i++ { + if got := string(items[i].([]byte)); got != strconv.Itoa(i+4) { + t.Errorf("items[%d] = %q, want %q", i, got, strconv.Itoa(i+4)) + } + } + + n, remaining, err = q.DequeueBatchWithAckIds(items, ackIDs) + if err != nil || n != 0 || remaining != 0 { + t.Fatalf("Empty batch: n=%d remaining=%d err=%v, want 0, 0, nil", n, remaining, err) + } + if got := countRows(t, q.f.db, "status = 'processing'"); got != 10 { + t.Errorf("Expected 10 processing rows, got %d", got) + } +} + +func TestBatchMinLen(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 10) + + n, remaining, err := q.DequeueBatchWithAckIds(make([]any, 5), make([]string, 2)) + if err != nil || n > 2 || remaining != 10-n { + t.Fatalf("n=%d remaining=%d err=%v, want n <= 2, 10-n, nil", n, remaining, err) + } + if got := countRows(t, q.f.db, "status = 'processing'"); got != n { + t.Errorf("Expected %d processing rows, got %d", n, got) + } +} + +func TestBatchRemainingCap(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 300) + + n, remaining, err := q.DequeueBatchWithAckIds(make([]any, 1), make([]string, 1)) + if err != nil || n != 1 || remaining != remainingCap { + t.Fatalf("n=%d remaining=%d err=%v, want 1, %d, nil", n, remaining, err, remainingCap) + } + if got := q.Len(); got != 299 { + t.Errorf("Len() = %d, want 299", got) + } +} + +func TestBatchClosed(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 3) + q.Close() + + n, _, err := q.DequeueBatchWithAckIds(make([]any, 3), make([]string, 3)) + if n != 0 || err != nil { + t.Fatalf("n=%d err=%v after Close, want 0, nil", n, err) + } + if got := countRows(t, q.f.db, "status = 'pending'"); got != 3 { + t.Errorf("Expected 3 pending rows, got %d", got) + } +} + +func TestCrashAfterBatch(t *testing.T) { + dir := t.TempDir() + qs := mustNew(t, dir) + q, err := qs.NewQueue("q") + if err != nil { + t.Fatalf("Failed to create queue: %v", err) + } + enqueueN(t, q, 5) + if n, _, err := q.DequeueBatchWithAckIds(make([]any, 3), make([]string, 3)); n != 3 || err != nil { + t.Fatalf("Batch of 3: n=%d err=%v", n, err) + } + // The process "dies": the file closes with 3 rows claimed and no ack + if err := qs.Close(); err != nil { + t.Fatalf("Failed to close: %v", err) + } + + qs = mustNew(t, dir) + defer qs.Close() + q, err = qs.NewQueue("q") + if err != nil { + t.Fatalf("Failed to reopen queue: %v", err) + } + if got := q.Len(); got != 5 { + t.Fatalf("Len() = %d after reopen, want 5", got) + } + for i := 0; i < 5; i++ { + item, ok := q.Dequeue() + if !ok { + t.Fatalf("Dequeue %d failed", i) + } + if got := string(item.([]byte)); got != strconv.Itoa(i) { + t.Fatalf("Dequeue %d = %q, want %q", i, got, strconv.Itoa(i)) + } + } +} + +func TestConcurrentWritersOneFile(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + const producers, perProducer, ackers, batch = 8, 500, 16, 16 + const total = producers * perProducer + + var pw sync.WaitGroup + for p := 0; p < producers; p++ { + pw.Add(1) + go func(p int) { + defer pw.Done() + for i := 0; i < perProducer; i++ { + if !q.Enqueue([]byte(fmt.Sprintf("%d-%d", p, i))) { + t.Errorf("Enqueue %d-%d failed", p, i) + } + } + }(p) + } + + type job struct{ data, ackID string } + jobs := make(chan job, total) + var ( + mu sync.Mutex + seen = map[string]int{} + ) + drained := make(chan struct{}) + go func() { + defer close(drained) + defer close(jobs) + items, ackIDs := make([]any, batch), make([]string, batch) + for got := 0; got < total; { + n, _, err := q.DequeueBatchWithAckIds(items, ackIDs) + if err != nil { + t.Errorf("Batch pop failed: %v", err) + return + } + for i := 0; i < n; i++ { + jobs <- job{string(items[i].([]byte)), ackIDs[i]} + } + got += n + } + }() + + var aw sync.WaitGroup + for a := 0; a < ackers; a++ { + aw.Add(1) + go func() { + defer aw.Done() + for j := range jobs { + mu.Lock() + seen[j.data]++ + mu.Unlock() + if !q.Acknowledge(j.ackID) { + t.Errorf("Acknowledge(%s) of %s failed", j.ackID, j.data) + } + } + }() + } + + pw.Wait() + <-drained + aw.Wait() + + if len(seen) != total { + t.Errorf("Expected %d distinct jobs, got %d", total, len(seen)) + } + for data, n := range seen { + if n != 1 { + t.Errorf("Job %s delivered %d times", data, n) + } + } + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Errorf("Expected no rows left, got %d", n) + } +} + +// TestWritesTakeFileMutex holds the file's write mutex and checks that every +// write waits for it and no read does. +func TestWritesTakeFileMutex(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 10) + _, _, ack1 := q.DequeueWithAckId() + + writes := []struct { + name string + op func() + }{ + {"Enqueue", func() { q.Enqueue([]byte("x")) }}, + {"Dequeue", func() { q.Dequeue() }}, + {"DequeueWithAckId", func() { q.DequeueWithAckId() }}, + {"DequeueBatchWithAckIds", func() { q.DequeueBatchWithAckIds(make([]any, 2), make([]string, 2)) }}, + {"Acknowledge", func() { q.Acknowledge(ack1) }}, + {"requeueNoAck", func() { q.requeueNoAck() }}, + {"Purge", func() { q.Purge() }}, + } + for _, w := range writes { + t.Run(w.name, func(t *testing.T) { + q.f.mu.Lock() + done := make(chan struct{}) + go func() { + w.op() + close(done) + }() + select { + case <-done: + q.f.mu.Unlock() + t.Fatalf("%s ran while the write mutex was held", w.name) + case <-time.After(100 * time.Millisecond): + } + q.f.mu.Unlock() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("%s did not finish after the mutex was released", w.name) + } + }) + } + + reads := []struct { + name string + op func() + }{ + {"Len", func() { q.Len() }}, + {"Values", func() { q.Values() }}, + } + for _, r := range reads { + t.Run(r.name, func(t *testing.T) { + q.f.mu.Lock() + defer q.f.mu.Unlock() + done := make(chan struct{}) + go func() { + r.op() + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("%s waited for the write mutex", r.name) + } + }) + } +} diff --git a/registry.go b/registry.go index 13f3652..355a8bc 100644 --- a/registry.go +++ b/registry.go @@ -17,7 +17,7 @@ type file struct { db *sql.DB pages int // checkpoint pages it was opened with refs int // Queues values holding it; under registry.mu - mu sync.Mutex // write mutex (used from Task 6) + mu sync.Mutex // write mutex: every write on the file holds it, reads do not handles map[string]*handle // lower-cased queue name -> handle; under registry.mu recovered map[string]bool // lower-cased table -> recovery done; under registry.mu } From 7b6ca6fae580f34d0793d4a6b32ed226cbbfcb1c Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 12:26:25 +0600 Subject: [PATCH 2/3] docs: batch pops, the per-file write lock, measured WAL peaks --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f989d3c..3e9975f 100644 --- a/README.md +++ b/README.md @@ -156,22 +156,21 @@ On case-insensitive filesystems (macOS, Windows), spell a path the same way ever ## Checkpoints -SQLite appends writes to a write-ahead log (WAL) and folds them back into the database file in a checkpoint. `WithCheckpointPages(n)` sets how many pages the WAL may reach before SQLite checkpoints on its own. The default is 1000 pages, SQLite's own setting, which keeps the WAL near 4 MB per file. With it, an `Enqueue` occasionally waits for a checkpoint's disk sync. +SQLite appends writes to a write-ahead log (WAL) and folds them back into the database file in a checkpoint. `WithCheckpointPages(n)` sets how many pages the WAL may reach before SQLite checkpoints on its own. The default is 1000 pages, SQLite's own setting. A page is 4 KB, so a checkpoint starts once the WAL reaches about 4 MB. That size is where a checkpoint is triggered, not a hard limit on the file. With the default, an `Enqueue` occasionally waits for a checkpoint's disk sync. ```go queuesManager, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) ``` -Raising it makes those waits rarer, which cuts `Enqueue` tail latency and raises throughput, at the price of a larger WAL on disk. For example: +Raising it makes those waits rarer, which cuts `Enqueue` tail latency and raises throughput, at the price of a larger WAL on disk: 10000 pages triggers a checkpoint at about 41 MB. -- 4000 pages: the WAL peaks near 16 MB per file. -- 10000 pages: the WAL peaks near 41 MB per file. +Measured on one machine, in a 10-minute soak of one queue in one file (4 producers adding 20,000 jobs per round, 16 workers, 1.9 to 4 million jobs in all), the WAL peaked at 4.2 MB with 1000 pages and at 41 MB with 10000 pages. The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. ## Compatibility -sqliteq does not import varmq. Batch pops and `EnqueueOnce` are being added for v0.3 and need varmq ≥ v1.10.0. +sqliteq does not import varmq. Queues and priority queues implement `DequeueBatchWithAckIds`, so with varmq ≥ v1.10.0 a worker claims several jobs in one statement. An older varmq never calls it and claims one job per call, which still works. `EnqueueOnce` comes in a later release. ## How It Works @@ -192,6 +191,8 @@ SQLiteQ uses a SQLite database to store queue items with the following schema: - Operations leverage SQLite's indexing for logarithmic time complexity rather than true constant-time - SQLite's WAL (Write-Ahead Logging) mode is enabled for better concurrent access - One partial index covers the pending items in queue order; opening a v0.2 table replaces its old indexes with it +- Writes to one file (adds, claims, acks, purges) take one lock in the process, so they wait for each other in Go rather than in SQLite's busy handler. `Len` and `Values` do not take it +- A batch pop reports the items left after it, counting at most 256; `Len` gives the exact count ## 👤 Author From 3403a26e315ae1509dca11f2d52f3230edc3c9b0 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 12:36:48 +0600 Subject: [PATCH 3/3] docs: quote WAL peaks from the published benchmark's walMB metric --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3e9975f..4ff8415 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ On case-insensitive filesystems (macOS, Windows), spell a path the same way ever ## Checkpoints -SQLite appends writes to a write-ahead log (WAL) and folds them back into the database file in a checkpoint. `WithCheckpointPages(n)` sets how many pages the WAL may reach before SQLite checkpoints on its own. The default is 1000 pages, SQLite's own setting. A page is 4 KB, so a checkpoint starts once the WAL reaches about 4 MB. That size is where a checkpoint is triggered, not a hard limit on the file. With the default, an `Enqueue` occasionally waits for a checkpoint's disk sync. +SQLite appends writes to a write-ahead log (WAL) and folds them back into the database file in a checkpoint. `WithCheckpointPages(n)` sets how many pages the WAL may reach before SQLite checkpoints on its own. The default is 1000 pages, SQLite's own setting. A page is 4 KB, so a checkpoint starts once the WAL reaches about 4 MB. That size is where a checkpoint is triggered, not a hard limit on the file. A checkpoint runs inside the commit of the write that crosses that size, while that write holds the file's write lock. So with the default, any write on that file (an `Enqueue`, a claim, an ack or a purge) occasionally waits for a checkpoint's disk sync. ```go queuesManager, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) @@ -164,7 +164,7 @@ queuesManager, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) Raising it makes those waits rarer, which cuts `Enqueue` tail latency and raises throughput, at the price of a larger WAL on disk: 10000 pages triggers a checkpoint at about 41 MB. -Measured on one machine, in a 10-minute soak of one queue in one file (4 producers adding 20,000 jobs per round, 16 workers, 1.9 to 4 million jobs in all), the WAL peaked at 4.2 MB with 1000 pages and at 41 MB with 10000 pages. +In the `Mixed1Q` and `Drain1Q` rows of the [varmq-benchmarks `sqlite` benchmark](https://github.com/goptics/varmq-benchmarks/tree/main/sqlite) (one queue in one file, 20,000 jobs), its `walMB` metric, the largest size the WAL file reached, was 4.1 MB with 1000 pages and 41 MB with 10000 pages, measured on one machine. The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this.