diff --git a/README.md b/README.md index f989d3c..4ff8415 100644 --- a/README.md +++ b/README.md @@ -156,22 +156,21 @@ On case-insensitive filesystems (macOS, Windows), spell a path the same way ever ## Checkpoints -SQLite appends writes to a write-ahead log (WAL) and folds them back into the database file in a checkpoint. `WithCheckpointPages(n)` sets how many pages the WAL may reach before SQLite checkpoints on its own. The default is 1000 pages, SQLite's own setting, which keeps the WAL near 4 MB per file. With it, an `Enqueue` occasionally waits for a checkpoint's disk sync. +SQLite appends writes to a write-ahead log (WAL) and folds them back into the database file in a checkpoint. `WithCheckpointPages(n)` sets how many pages the WAL may reach before SQLite checkpoints on its own. The default is 1000 pages, SQLite's own setting. A page is 4 KB, so a checkpoint starts once the WAL reaches about 4 MB. That size is where a checkpoint is triggered, not a hard limit on the file. A checkpoint runs inside the commit of the write that crosses that size, while that write holds the file's write lock. So with the default, any write on that file (an `Enqueue`, a claim, an ack or a purge) occasionally waits for a checkpoint's disk sync. ```go queuesManager, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) ``` -Raising it makes those waits rarer, which cuts `Enqueue` tail latency and raises throughput, at the price of a larger WAL on disk. For example: +Raising it makes those waits rarer, which cuts `Enqueue` tail latency and raises throughput, at the price of a larger WAL on disk: 10000 pages triggers a checkpoint at about 41 MB. -- 4000 pages: the WAL peaks near 16 MB per file. -- 10000 pages: the WAL peaks near 41 MB per file. +In the `Mixed1Q` and `Drain1Q` rows of the [varmq-benchmarks `sqlite` benchmark](https://github.com/goptics/varmq-benchmarks/tree/main/sqlite) (one queue in one file, 20,000 jobs), its `walMB` metric, the largest size the WAL file reached, was 4.1 MB with 1000 pages and 41 MB with 10000 pages, measured on one machine. The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. ## Compatibility -sqliteq does not import varmq. Batch pops and `EnqueueOnce` are being added for v0.3 and need varmq ≥ v1.10.0. +sqliteq does not import varmq. Queues and priority queues implement `DequeueBatchWithAckIds`, so with varmq ≥ v1.10.0 a worker claims several jobs in one statement. An older varmq never calls it and claims one job per call, which still works. `EnqueueOnce` comes in a later release. ## How It Works @@ -192,6 +191,8 @@ SQLiteQ uses a SQLite database to store queue items with the following schema: - Operations leverage SQLite's indexing for logarithmic time complexity rather than true constant-time - SQLite's WAL (Write-Ahead Logging) mode is enabled for better concurrent access - One partial index covers the pending items in queue order; opening a v0.2 table replaces its old indexes with it +- Writes to one file (adds, claims, acks, purges) take one lock in the process, so they wait for each other in Go rather than in SQLite's busy handler. `Len` and `Values` do not take it +- A batch pop reports the items left after it, counting at most 256; `Len` gives the exact count ## 👤 Author diff --git a/priority_queue_test.go b/priority_queue_test.go index a80c8fb..39b2ed7 100644 --- a/priority_queue_test.go +++ b/priority_queue_test.go @@ -623,3 +623,33 @@ func TestPriorityOrderByID(t *testing.T) { } } } + +func TestBatchPriorityOrder(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("q") + if err != nil { + t.Fatalf("Failed to create priority queue: %v", err) + } + + // Job names are "/" + for i, p := range []int{3, 1, 1, 0, 2} { + if !pq.Enqueue([]byte(fmt.Sprintf("%d/%d", p, i)), p) { + t.Fatalf("Enqueue %d failed", i) + } + } + + items, ackIDs := make([]any, 5), make([]string, 5) + n, remaining, err := pq.DequeueBatchWithAckIds(items, ackIDs) + if err != nil || n != 5 || remaining != 0 { + t.Fatalf("n=%d remaining=%d err=%v, want 5, 0, nil", n, remaining, err) + } + for i, want := range []string{"0/3", "1/1", "1/2", "2/4", "3/0"} { + if got := string(items[i].([]byte)); got != want { + t.Errorf("items[%d] = %q, want %q", i, got, want) + } + if !pq.Acknowledge(ackIDs[i]) { + t.Errorf("Acknowledge(%s) failed", ackIDs[i]) + } + } +} diff --git a/queue.go b/queue.go index 31fbd20..90ac89c 100644 --- a/queue.go +++ b/queue.go @@ -58,11 +58,8 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro } // The db is shared by every queue of the file, so it is not closed on errors - if err := q.initTable(); err != nil { - return nil, fmt.Errorf("failed to initialize table: %w", err) - } - if err := q.migrate(); err != nil { - return nil, fmt.Errorf("failed to migrate indexes: %w", err) + if err := q.initSchema(); err != nil { + return nil, err } if err := q.prepare(); err != nil { return nil, fmt.Errorf("failed to prepare statements: %w", err) @@ -71,6 +68,20 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro return q, nil } +// initSchema creates or migrates the table and its index under the write mutex +func (q *Queue) initSchema() error { + q.f.mu.Lock() + defer q.f.mu.Unlock() + + if err := q.initTable(); err != nil { + return fmt.Errorf("failed to initialize table: %w", err) + } + if err := q.migrate(); err != nil { + return fmt.Errorf("failed to migrate indexes: %w", err) + } + return nil +} + // initTable creates the table as v0.2 defines it, so old tables need no // rebuild, and adds the priority column to a priority queue's table when // it is missing @@ -200,6 +211,9 @@ func (q *Queue) closeStmts() { // requeueNoAck puts rows left in processing without an ack back to pending func (q *Queue) requeueNoAck() error { + q.f.mu.Lock() + defer q.f.mu.Unlock() + _, err := q.f.db.Exec( fmt.Sprintf("UPDATE %s SET status = 'pending', updated_at = ? WHERE status = 'processing' AND ack = 0", q.table), time.Now().UTC(), @@ -258,6 +272,9 @@ func (q *Queue) enqueue(item any, priority int) bool { } now := time.Now().UTC() + q.f.mu.Lock() + defer q.f.mu.Unlock() + var err error if q.priority { _, err = q.insert.Exec(item, now, now, priority) @@ -270,6 +287,26 @@ func (q *Queue) enqueue(item any, priority int) bool { // claimN moves up to n pending rows to processing in one statement and // returns them in queue order func (q *Queue) claimN(n int) ([]claimed, error) { + q.f.mu.Lock() + out, err := q.claimRows(n) + q.f.mu.Unlock() + if err != nil { + return nil, err + } + + // RETURNING order is unspecified + slices.SortFunc(out, func(a, b claimed) int { + if c := cmp.Compare(a.priority, b.priority); c != 0 { + return c + } + return cmp.Compare(a.id, b.id) + }) + return out, nil +} + +// claimRows runs the claim statement and reads every row it returns; the +// caller holds the write mutex +func (q *Queue) claimRows(n int) ([]claimed, error) { rows, err := q.claim.Query(time.Now().UTC(), n) if err != nil { return nil, err @@ -290,18 +327,7 @@ func (q *Queue) claimN(n int) ([]claimed, error) { out = append(out, c) } // Reading to the end runs the statement to completion, which commits it - if err := rows.Err(); err != nil { - return nil, err - } - - // RETURNING order is unspecified - slices.SortFunc(out, func(a, b claimed) int { - if c := cmp.Compare(a.priority, b.priority); c != 0 { - return c - } - return cmp.Compare(a.id, b.id) - }) - return out, nil + return out, rows.Err() } // Dequeue removes and returns the next item from the queue @@ -311,6 +337,10 @@ func (q *Queue) Dequeue() (any, bool) { return nil, false } + q.f.mu.Lock() + defer q.f.mu.Unlock() + + // Deferred last, so the rows close (and the statement ends) before unlocking rows, err := q.pop.Query() if err != nil { return nil, false @@ -345,6 +375,36 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) { return c[0].data, true, strconv.FormatInt(c[0].id, 10) } +// DequeueBatchWithAckIds implements varmq.IBatchAcknowledgeable: it moves up +// to min(len(items), len(ackIDs)) items to processing in one statement, in +// queue order, each under its row id as ack id. remaining is the pending +// count after the move, capped at remainingCap. n is 0 with a nil error when +// the queue is empty or closed. +func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaining int, err error) { + want := min(len(items), len(ackIDs)) + if q.closed.Load() || want == 0 { + return 0, 0, nil + } + + c, err := q.claimN(want) + if err != nil { + return 0, 0, err + } + if len(c) == 0 { + return 0, 0, nil + } + for i, row := range c { + items[i], ackIDs[i] = row.data, strconv.FormatInt(row.id, 10) + } + + // The rows are claimed already: an error here would make varmq drop them, + // so a failed count reports items left and the next pop finds out + if err := q.countCapped.QueryRow().Scan(&remaining); err != nil { + return len(c), 1, nil + } + return len(c), remaining, nil +} + // Acknowledge marks a processing item as completed, deleting it unless // WithRemoveOnComplete(false) is set // Returns true only if exactly one processing row had this ack ID @@ -354,6 +414,9 @@ func (q *Queue) Acknowledge(ackID string) bool { return false } + q.f.mu.Lock() + defer q.f.mu.Unlock() + var ( result sql.Result err error @@ -405,6 +468,9 @@ func (q *Queue) Values() []any { // Purge removes all items from the queue func (q *Queue) Purge() { + q.f.mu.Lock() + defer q.f.mu.Unlock() + q.f.db.Exec(fmt.Sprintf("DELETE FROM %s", q.table)) } diff --git a/queue_test.go b/queue_test.go index 023bbdd..fb081e8 100644 --- a/queue_test.go +++ b/queue_test.go @@ -8,6 +8,7 @@ import ( "sync" "sync/atomic" "testing" + "time" ) func TestSQLiteQueue(t *testing.T) { @@ -594,3 +595,289 @@ func TestConcurrentClaimsNoDuplicates(t *testing.T) { t.Errorf("Expected no rows left, got %d", n) } } + +func TestImplementsBatch(t *testing.T) { + type batcher interface { + DequeueBatchWithAckIds([]any, []string) (int, int, error) + } + var _ batcher = (*Queue)(nil) + var _ batcher = (*PriorityQueue)(nil) +} + +// enqueueN adds jobs "0".."n-1" to q. +func enqueueN(t *testing.T, q *Queue, n int) { + t.Helper() + for i := 0; i < n; i++ { + if !q.Enqueue([]byte(strconv.Itoa(i))) { + t.Fatalf("Enqueue %d failed", i) + } + } +} + +func TestBatchPop(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 10) + + items, ackIDs := make([]any, 4), make([]string, 4) + n, remaining, err := q.DequeueBatchWithAckIds(items, ackIDs) + if err != nil || n != 4 || remaining != 6 { + t.Fatalf("Batch of 4: n=%d remaining=%d err=%v, want 4, 6, nil", n, remaining, err) + } + seen := map[string]bool{} + for i := 0; i < n; i++ { + if got := string(items[i].([]byte)); got != strconv.Itoa(i) { + t.Errorf("items[%d] = %q, want %q", i, got, strconv.Itoa(i)) + } + if _, ok := parseAckID(ackIDs[i]); !ok { + t.Errorf("ackIDs[%d] = %q is not a row id", i, ackIDs[i]) + } + if seen[ackIDs[i]] { + t.Errorf("ackIDs[%d] = %q repeats", i, ackIDs[i]) + } + seen[ackIDs[i]] = true + } + + items, ackIDs = make([]any, 10), make([]string, 10) + n, remaining, err = q.DequeueBatchWithAckIds(items, ackIDs) + if err != nil || n != 6 || remaining != 0 { + t.Fatalf("Batch of 10: n=%d remaining=%d err=%v, want 6, 0, nil", n, remaining, err) + } + for i := 0; i < n; i++ { + if got := string(items[i].([]byte)); got != strconv.Itoa(i+4) { + t.Errorf("items[%d] = %q, want %q", i, got, strconv.Itoa(i+4)) + } + } + + n, remaining, err = q.DequeueBatchWithAckIds(items, ackIDs) + if err != nil || n != 0 || remaining != 0 { + t.Fatalf("Empty batch: n=%d remaining=%d err=%v, want 0, 0, nil", n, remaining, err) + } + if got := countRows(t, q.f.db, "status = 'processing'"); got != 10 { + t.Errorf("Expected 10 processing rows, got %d", got) + } +} + +func TestBatchMinLen(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 10) + + n, remaining, err := q.DequeueBatchWithAckIds(make([]any, 5), make([]string, 2)) + if err != nil || n > 2 || remaining != 10-n { + t.Fatalf("n=%d remaining=%d err=%v, want n <= 2, 10-n, nil", n, remaining, err) + } + if got := countRows(t, q.f.db, "status = 'processing'"); got != n { + t.Errorf("Expected %d processing rows, got %d", n, got) + } +} + +func TestBatchRemainingCap(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 300) + + n, remaining, err := q.DequeueBatchWithAckIds(make([]any, 1), make([]string, 1)) + if err != nil || n != 1 || remaining != remainingCap { + t.Fatalf("n=%d remaining=%d err=%v, want 1, %d, nil", n, remaining, err, remainingCap) + } + if got := q.Len(); got != 299 { + t.Errorf("Len() = %d, want 299", got) + } +} + +func TestBatchClosed(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 3) + q.Close() + + n, _, err := q.DequeueBatchWithAckIds(make([]any, 3), make([]string, 3)) + if n != 0 || err != nil { + t.Fatalf("n=%d err=%v after Close, want 0, nil", n, err) + } + if got := countRows(t, q.f.db, "status = 'pending'"); got != 3 { + t.Errorf("Expected 3 pending rows, got %d", got) + } +} + +func TestCrashAfterBatch(t *testing.T) { + dir := t.TempDir() + qs := mustNew(t, dir) + q, err := qs.NewQueue("q") + if err != nil { + t.Fatalf("Failed to create queue: %v", err) + } + enqueueN(t, q, 5) + if n, _, err := q.DequeueBatchWithAckIds(make([]any, 3), make([]string, 3)); n != 3 || err != nil { + t.Fatalf("Batch of 3: n=%d err=%v", n, err) + } + // The process "dies": the file closes with 3 rows claimed and no ack + if err := qs.Close(); err != nil { + t.Fatalf("Failed to close: %v", err) + } + + qs = mustNew(t, dir) + defer qs.Close() + q, err = qs.NewQueue("q") + if err != nil { + t.Fatalf("Failed to reopen queue: %v", err) + } + if got := q.Len(); got != 5 { + t.Fatalf("Len() = %d after reopen, want 5", got) + } + for i := 0; i < 5; i++ { + item, ok := q.Dequeue() + if !ok { + t.Fatalf("Dequeue %d failed", i) + } + if got := string(item.([]byte)); got != strconv.Itoa(i) { + t.Fatalf("Dequeue %d = %q, want %q", i, got, strconv.Itoa(i)) + } + } +} + +func TestConcurrentWritersOneFile(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + const producers, perProducer, ackers, batch = 8, 500, 16, 16 + const total = producers * perProducer + + var pw sync.WaitGroup + for p := 0; p < producers; p++ { + pw.Add(1) + go func(p int) { + defer pw.Done() + for i := 0; i < perProducer; i++ { + if !q.Enqueue([]byte(fmt.Sprintf("%d-%d", p, i))) { + t.Errorf("Enqueue %d-%d failed", p, i) + } + } + }(p) + } + + type job struct{ data, ackID string } + jobs := make(chan job, total) + var ( + mu sync.Mutex + seen = map[string]int{} + ) + drained := make(chan struct{}) + go func() { + defer close(drained) + defer close(jobs) + items, ackIDs := make([]any, batch), make([]string, batch) + for got := 0; got < total; { + n, _, err := q.DequeueBatchWithAckIds(items, ackIDs) + if err != nil { + t.Errorf("Batch pop failed: %v", err) + return + } + for i := 0; i < n; i++ { + jobs <- job{string(items[i].([]byte)), ackIDs[i]} + } + got += n + } + }() + + var aw sync.WaitGroup + for a := 0; a < ackers; a++ { + aw.Add(1) + go func() { + defer aw.Done() + for j := range jobs { + mu.Lock() + seen[j.data]++ + mu.Unlock() + if !q.Acknowledge(j.ackID) { + t.Errorf("Acknowledge(%s) of %s failed", j.ackID, j.data) + } + } + }() + } + + pw.Wait() + <-drained + aw.Wait() + + if len(seen) != total { + t.Errorf("Expected %d distinct jobs, got %d", total, len(seen)) + } + for data, n := range seen { + if n != 1 { + t.Errorf("Job %s delivered %d times", data, n) + } + } + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Errorf("Expected no rows left, got %d", n) + } +} + +// TestWritesTakeFileMutex holds the file's write mutex and checks that every +// write waits for it and no read does. +func TestWritesTakeFileMutex(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 10) + _, _, ack1 := q.DequeueWithAckId() + + writes := []struct { + name string + op func() + }{ + {"Enqueue", func() { q.Enqueue([]byte("x")) }}, + {"Dequeue", func() { q.Dequeue() }}, + {"DequeueWithAckId", func() { q.DequeueWithAckId() }}, + {"DequeueBatchWithAckIds", func() { q.DequeueBatchWithAckIds(make([]any, 2), make([]string, 2)) }}, + {"Acknowledge", func() { q.Acknowledge(ack1) }}, + {"requeueNoAck", func() { q.requeueNoAck() }}, + {"Purge", func() { q.Purge() }}, + } + for _, w := range writes { + t.Run(w.name, func(t *testing.T) { + q.f.mu.Lock() + done := make(chan struct{}) + go func() { + w.op() + close(done) + }() + select { + case <-done: + q.f.mu.Unlock() + t.Fatalf("%s ran while the write mutex was held", w.name) + case <-time.After(100 * time.Millisecond): + } + q.f.mu.Unlock() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("%s did not finish after the mutex was released", w.name) + } + }) + } + + reads := []struct { + name string + op func() + }{ + {"Len", func() { q.Len() }}, + {"Values", func() { q.Values() }}, + } + for _, r := range reads { + t.Run(r.name, func(t *testing.T) { + q.f.mu.Lock() + defer q.f.mu.Unlock() + done := make(chan struct{}) + go func() { + r.op() + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("%s waited for the write mutex", r.name) + } + }) + } +} diff --git a/registry.go b/registry.go index 13f3652..355a8bc 100644 --- a/registry.go +++ b/registry.go @@ -17,7 +17,7 @@ type file struct { db *sql.DB pages int // checkpoint pages it was opened with refs int // Queues values holding it; under registry.mu - mu sync.Mutex // write mutex (used from Task 6) + mu sync.Mutex // write mutex: every write on the file holds it, reads do not handles map[string]*handle // lower-cased queue name -> handle; under registry.mu recovered map[string]bool // lower-cased table -> recovery done; under registry.mu }