From 7af7a1f70b8935b27e6b9e0da5127165a73c4026 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 16:54:48 +0600 Subject: [PATCH 1/3] feat: EnqueueOnce dedupes jobs by id while their row exists Queue.EnqueueOnce(item, id) and PriorityQueue.EnqueueOnce(item, priority, id) implement varmq's IDedupQueue and IDedupPriorityQueue. Opening a table adds a job_id TEXT column when it is missing and a unique partial index on it over the rows that have one. The insert ends in ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING, so a repeat stores nothing and returns true, while a NOT NULL failure such as Enqueue(nil) still fails. Enqueue and an empty id bind NULL. The index is __sqliteq_job_id rather than _job_id, for the reason the pending index has its suffix: tables and indexes share one namespace, and a queue named orders_job_id could not open beside orders. --- commit_test.go | 2 +- dedup_test.go | 381 ++++++++++++++++++++++++++++++++++++++++++++++ migration_test.go | 11 +- priority_queue.go | 6 + queue.go | 60 +++++++- 5 files changed, 446 insertions(+), 14 deletions(-) create mode 100644 dedup_test.go diff --git a/commit_test.go b/commit_test.go index b29a954..6b1a7e1 100644 --- a/commit_test.go +++ b/commit_test.go @@ -296,7 +296,7 @@ func TestCommitRoundFallsBackPerOp(t *testing.T) { defer qs.Close() add := func(tx *sql.Tx) (bool, error) { - _, err := tx.Stmt(q.insert).Exec([]byte("x"), time.Now(), time.Now()) + _, err := tx.Stmt(q.insert).Exec([]byte("x"), time.Now(), time.Now(), nil) return err == nil, err } bad := func(*sql.Tx) (bool, error) { return false, errors.New("bad op") } diff --git a/dedup_test.go b/dedup_test.go new file mode 100644 index 0000000..b2aee68 --- /dev/null +++ b/dedup_test.go @@ -0,0 +1,381 @@ +package sqliteq + +import ( + "fmt" + "path/filepath" + "sync" + "testing" +) + +// TestImplementsDedup checks at compile time that both kinds of queue +// implement varmq's IDedupQueue and IDedupPriorityQueue. +func TestImplementsDedup(t *testing.T) { + var _ interface{ EnqueueOnce(any, string) bool } = (*Queue)(nil) + var _ interface{ EnqueueOnce(any, int, string) bool } = (*PriorityQueue)(nil) +} + +// once adds item under id; on a priority queue it does so at priority 1 +type once func(item any, id string) bool + +// openOnce opens queue "q", or priority queue "q", in a fresh folder store, +// and returns its Queue with its EnqueueOnce. countRows reads table "q". +func openOnce(t *testing.T, priority bool, opts ...Option) (*Queue, once, Queues) { + t.Helper() + qs := mustNew(t, t.TempDir()) + if priority { + pq, err := qs.NewPriorityQueue("q", opts...) + if err != nil { + qs.Close() + t.Fatalf("Failed to create priority queue: %v", err) + } + return pq.Queue, func(item any, id string) bool { return pq.EnqueueOnce(item, 1, id) }, qs + } + q, err := qs.NewQueue("q", opts...) + if err != nil { + qs.Close() + t.Fatalf("Failed to create queue: %v", err) + } + return q, q.EnqueueOnce, qs +} + +// dequeueAll pops every pending item and returns them as strings, in order +func dequeueAll(t *testing.T, q *Queue) []string { + t.Helper() + var out []string + for { + item, ok := q.Dequeue() + if !ok { + return out + } + out = append(out, string(item.([]byte))) + } +} + +func TestEnqueueOnceRepeat(t *testing.T) { testOnceRepeat(t, false) } + +func testOnceRepeat(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("First EnqueueOnce failed") + } + if !add([]byte("b"), "j1") { + t.Error("A repeat returned false, want true") + } + if n := q.Len(); n != 1 { + t.Errorf("Expected 1 pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[a]" { + t.Errorf("Dequeued %v, want [a]", got) + } + + // Dequeue deleted the row, and the id with it + if !add([]byte("c"), "j1") { + t.Error("EnqueueOnce after Dequeue returned false, want true") + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c]" { + t.Errorf("Dequeued %v after the re-add, want [c]", got) + } +} + +func TestEnqueueOnceWhileProcessing(t *testing.T) { testOnceWhileProcessing(t, false) } + +func testOnceWhileProcessing(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { + t.Fatal("DequeueWithAckId failed") + } + + if !add([]byte("b"), "j1") { + t.Error("A repeat of a processing job returned false, want true") + } + if n := q.Len(); n != 0 { + t.Errorf("Expected no pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row (the processing one), got %d", n) + } +} + +func TestEnqueueOnceAfterAck(t *testing.T) { testOnceAfterAck(t, false) } + +func testOnceAfterAck(t *testing.T, priority bool) { + for _, remove := range []bool{true, false} { + t.Run(fmt.Sprintf("remove=%v", remove), func(t *testing.T) { + q, add, qs := openOnce(t, priority, WithRemoveOnComplete(remove)) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("EnqueueOnce failed") + } + _, ok, ackID := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId failed") + } + if !q.Acknowledge(ackID) { + t.Fatal("Acknowledge failed") + } + + if !add([]byte("c"), "j1") { + t.Error("EnqueueOnce after the ack returned false, want true") + } + if remove { + // The row is gone, and the id with it + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c]" { + t.Errorf("Dequeued %v, want [c]", got) + } + return + } + // The completed row keeps the id + if n := q.Len(); n != 0 { + t.Errorf("Expected no pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row (the completed one), got %d", n) + } + }) + } +} + +func TestEnqueueOncePriority(t *testing.T) { + t.Run("Repeat", func(t *testing.T) { testOnceRepeat(t, true) }) + t.Run("WhileProcessing", func(t *testing.T) { testOnceWhileProcessing(t, true) }) + t.Run("AfterAck", func(t *testing.T) { testOnceAfterAck(t, true) }) + + t.Run("Order", func(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("q") + if err != nil { + t.Fatalf("Failed to create priority queue: %v", err) + } + + for _, c := range []struct { + item string + priority int + id string + }{ + {"low", 5, "a"}, + {"high", 1, "b"}, + {"mid", 3, "c"}, + {"high-again", 0, "b"}, // a repeat: neither stored nor moved up + {"low-again", 9, "a"}, + {"no-id", 2, ""}, + } { + if !pq.EnqueueOnce([]byte(c.item), c.priority, c.id) { + t.Fatalf("EnqueueOnce(%s, %d, %q) failed", c.item, c.priority, c.id) + } + } + + if got := dequeueAll(t, pq.Queue); fmt.Sprint(got) != "[high no-id mid low]" { + t.Errorf("Dequeued %v, want [high no-id mid low]", got) + } + }) +} + +// An empty id stores the item like Enqueue does, every time +func TestEnqueueOnceEmptyID(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + for i := 0; i < 3; i++ { + if !q.EnqueueOnce([]byte("x"), "") { + t.Fatalf("EnqueueOnce %d with no id failed", i) + } + if !q.Enqueue([]byte("y")) { + t.Fatalf("Enqueue %d failed", i) + } + } + if n := q.Len(); n != 6 { + t.Errorf("Expected 6 pending items, got %d", n) + } + if n := countRows(t, q.f.db, "job_id IS NULL"); n != 6 { + t.Errorf("Expected 6 rows with a NULL job_id, got %d", n) + } +} + +// The conflict clause covers the job_id index only: a nil item still fails, +// and a failed add leaves its id free +func TestEnqueueOnceNil(t *testing.T) { + for _, priority := range []bool{false, true} { + t.Run(fmt.Sprintf("priority=%v", priority), func(t *testing.T) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if add(nil, "j1") { + t.Error("EnqueueOnce(nil) succeeded") + } + if q.Enqueue(nil) { + t.Error("Enqueue(nil) succeeded") + } + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Errorf("Expected no rows, got %d", n) + } + if !add([]byte("a"), "j1") { + t.Error("EnqueueOnce after a failed one failed") + } + if n := q.Len(); n != 1 { + t.Errorf("Expected 1 pending item, got %d", n) + } + }) + } +} + +// Two adds of one id in one grouped round: the second conflicts inside the +// round's transaction, both return true, and one row is stored. With a nil +// item in the round, the round falls back to one transaction per call and +// the same holds. +func TestEnqueueOnceOneRound(t *testing.T) { + for _, bad := range []bool{false, true} { + t.Run(fmt.Sprintf("bad=%v", bad), func(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + first := func() bool { return q.Enqueue([]byte("x")) } + rest := []func() bool{ + func() bool { return q.EnqueueOnce([]byte("a"), "j1") }, + func() bool { return q.EnqueueOnce([]byte("b"), "j1") }, + func() bool { return q.EnqueueOnce([]byte("c"), "j2") }, + } + if bad { + rest = append(rest, + func() bool { return q.Enqueue(nil) }, + func() bool { return q.EnqueueOnce(nil, "j3") }, + ) + } + + got := oneRound(t, q.f, first, rest) + for i := 0; i < 4; i++ { + if !got[i] { + t.Errorf("Call %d failed", i) + } + } + for i := 4; i < len(got); i++ { + if got[i] { + t.Errorf("Call %d (a nil item) succeeded", i) + } + } + if n := countRows(t, q.f.db, "job_id = 'j1'"); n != 1 { + t.Errorf("Expected 1 row for j1, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 3 { + t.Errorf("Expected 3 rows (x, j1, j2), got %d", n) + } + }) + } +} + +// 32 concurrent adds of one id: every call returns true and one row is stored +func TestEnqueueOnceConcurrent(t *testing.T) { + for _, priority := range []bool{false, true} { + t.Run(fmt.Sprintf("priority=%v", priority), func(t *testing.T) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + const n = 32 + var ( + wg sync.WaitGroup + start = make(chan struct{}) + oks [n]bool + ) + for g := 0; g < n; g++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + oks[g] = add([]byte(fmt.Sprint(g)), "j1") + }() + } + close(start) + wg.Wait() + + for g, ok := range oks { + if !ok { + t.Errorf("EnqueueOnce in goroutine %d returned false", g) + } + } + if l := q.Len(); l != 1 { + t.Errorf("Expected 1 pending item, got %d", l) + } + if c := countRows(t, q.f.db, "1"); c != 1 { + t.Errorf("Expected 1 row, got %d", c) + } + }) + } +} + +// A v0.2.3 file gains job_id and its unique partial index on open, keeps +// them on reopen, and its old rows still dequeue in order. +func TestEnqueueOnceOldFile(t *testing.T) { + for _, priority := range []bool{false, true} { + name := "jobs" + if priority { + name = "pjobs" + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "v023.db") + writeV023File(t, path, name, priority) + + for round := 1; round <= 2; round++ { + qs := mustNew(t, path, WithSingleFile()) + var ( + q *Queue + add once + ) + if priority { + pq, err := qs.NewPriorityQueue(name) + if err != nil { + t.Fatalf("Round %d: failed to open v0.2.3 priority queue: %v", round, err) + } + q, add = pq.Queue, func(item any, id string) bool { return pq.EnqueueOnce(item, 0, id) } + } else { + var err error + if q, err = qs.NewQueue(name); err != nil { + t.Fatalf("Round %d: failed to open v0.2.3 queue: %v", round, err) + } + add = q.EnqueueOnce + } + + if ok, err := hasColumn(q.f.db, name, "job_id"); err != nil || !ok { + t.Fatalf("Round %d: job_id column present = %v, %v", round, ok, err) + } + var unique, partial int + err := q.f.db.QueryRow(`SELECT "unique", partial FROM pragma_index_list(?) WHERE name = ?`, + name, name+jobIDIndexSuffix).Scan(&unique, &partial) + if err != nil || unique != 1 || partial != 1 { + t.Errorf("Round %d: job_id index unique=%d partial=%d err=%v, want a unique partial index", round, unique, partial, err) + } + if cols, err := indexColumns(q.f.db, name+jobIDIndexSuffix); err != nil || fmt.Sprint(cols) != "[job_id]" { + t.Errorf("Round %d: job_id index columns = %v, %v; want [job_id]", round, cols, err) + } + + if round == 1 { + if !add([]byte("new"), "j1") || !add([]byte("dup"), "j1") { + t.Error("EnqueueOnce on the old table failed") + } + if n := q.Len(); n != 6 { + t.Errorf("Expected 6 pending jobs, got %d", n) + } + } else { + want := "[job-1 job-2 job-3 job-4 job-5 new]" + if got := dequeueAll(t, q); fmt.Sprint(got) != want { + t.Errorf("Dequeued %v, want %s", got, want) + } + } + + if err := qs.Close(); err != nil { + t.Fatalf("Round %d: failed to close: %v", round, err) + } + } + }) + } +} diff --git a/migration_test.go b/migration_test.go index affe32f..bc94204 100644 --- a/migration_test.go +++ b/migration_test.go @@ -125,7 +125,7 @@ func TestMigrateV02File(t *testing.T) { dequeue, length, db = q.Dequeue, q.Len, q.f.db } - want := []string{name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} + want := []string{name + jobIDIndexSuffix, name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { t.Errorf("Round %d: indexes = %v, want %v", round, got, want) } @@ -207,11 +207,12 @@ func TestPendingIndexFollowsKind(t *testing.T) { } } -// Tables and indexes share one namespace in SQLite, so the pending index -// must not take a name a queue could have: a file with queues "orders" and -// "orders_pending" opens in either order, and again on reopen. +// Tables and indexes share one namespace in SQLite, so the pending and +// job_id indexes must not take a name a queue could have: a file with queues +// "orders", "orders_pending" and "orders_job_id" opens in either order, and +// again on reopen. func TestPendingIndexNameFreeForQueues(t *testing.T) { - for _, names := range [][]string{{"orders", "orders_pending"}, {"orders_pending", "orders"}} { + for _, names := range [][]string{{"orders", "orders_pending", "orders_job_id"}, {"orders_job_id", "orders_pending", "orders"}} { t.Run(names[0]+"_first", func(t *testing.T) { path := filepath.Join(t.TempDir(), "shared.db") for round := 1; round <= 2; round++ { diff --git a/priority_queue.go b/priority_queue.go index bc2ff32..0aeae49 100644 --- a/priority_queue.go +++ b/priority_queue.go @@ -22,3 +22,9 @@ func newPriorityQueue(f *file, name string, opts ...Option) (*PriorityQueue, err func (pq *PriorityQueue) Enqueue(item any, priority int) bool { return pq.enqueue(item, priority, "") } + +// EnqueueOnce implements varmq.IDedupPriorityQueue: Queue.EnqueueOnce with a +// priority. A repeat leaves the stored item and its priority as they are. +func (pq *PriorityQueue) EnqueueOnce(item any, priority int, id string) bool { + return pq.enqueue(item, priority, id) +} diff --git a/queue.go b/queue.go index bdcfa9f..c2f62a3 100644 --- a/queue.go +++ b/queue.go @@ -80,6 +80,9 @@ func (q *Queue) initSchema() error { if err := q.migrate(); err != nil { return fmt.Errorf("failed to migrate indexes: %w", err) } + if err := q.addJobID(); err != nil { + return fmt.Errorf("failed to add job_id: %w", err) + } return nil } @@ -110,9 +113,13 @@ func (q *Queue) initTable() error { return err } -// pendingIndexSuffix names a queue's partial pending index. Tables and -// indexes share one namespace, so the suffix is one no queue name would take. -const pendingIndexSuffix = "__sqliteq_pending" +// pendingIndexSuffix and jobIDIndexSuffix name a queue's partial indexes. +// Tables and indexes share one namespace, so the suffixes are ones no queue +// name would take. +const ( + pendingIndexSuffix = "__sqliteq_pending" + jobIDIndexSuffix = "__sqliteq_job_id" +) // migrate drops the v0.2 indexes and makes sure the one partial index on // pending rows matches the queue order. On a migrated table it changes nothing. @@ -142,6 +149,24 @@ func (q *Queue) migrate() error { return err } +// addJobID adds the job_id column EnqueueOnce fills, when it is missing, and +// its unique index over the rows that have one. Rows without an id, old ones +// and those of Enqueue, are NULL and outside the index. +func (q *Queue) addJobID() error { + exists, err := hasColumn(q.f.db, q.name, "job_id") + if err != nil { + return err + } + if !exists { + if _, err := q.f.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN job_id TEXT", q.table)); err != nil { + return err + } + } + _, err = q.f.db.Exec(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (job_id) WHERE job_id IS NOT NULL", + quoteIdent(q.name+jobIDIndexSuffix), q.table)) + return err +} + // indexColumns lists an index's columns in order; none if it does not exist func indexColumns(db *sql.DB, index string) ([]string, error) { rows, err := db.Query("SELECT name FROM pragma_index_info(?) ORDER BY seqno", index) @@ -170,7 +195,7 @@ func (q *Queue) prepare() (err error) { } }() - cols, vals, returning := "data, status, ack, created_at, updated_at", "?, 'pending', 0, ?, ?", "id, data" + cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data" if q.priority { cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data" } @@ -185,7 +210,9 @@ func (q *Queue) prepare() (err error) { stmt **sql.Stmt query string }{ - {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", q.table, cols, vals)}, + // The conflict target is the job_id index alone, so a NOT NULL or any + // other failure still errors; it must repeat the index's WHERE + {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING", q.table, cols, vals)}, {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)}, {&q.ack, ack}, @@ -266,8 +293,17 @@ func (q *Queue) Enqueue(item any) bool { return q.enqueue(item, 0, "") } +// EnqueueOnce implements varmq.IDedupQueue: it adds item under id unless a +// row with that id exists, pending, processing, or completed with +// WithRemoveOnComplete(false). A repeat stores nothing and returns true. An +// empty id behaves like Enqueue. +func (q *Queue) EnqueueOnce(item any, id string) bool { + return q.enqueue(item, 0, id) +} + // enqueue inserts one pending row in the file's next group commit round; -// priority is stored on priority queues only. id is unused for now. +// priority is stored on priority queues only. A row with a non-empty id is +// skipped, still with true, when a row with that id exists. func (q *Queue) enqueue(item any, priority int, id string) bool { if q.closed.Load() { return false @@ -280,18 +316,26 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { return false } + // No id is NULL, which the job_id index leaves out; "" would be one id + var jobID any + if id != "" { + jobID = id + } + now := time.Now().UTC() return q.f.b.do(func(tx *sql.Tx) (bool, error) { insert := q.insert if tx != nil { insert = tx.Stmt(insert) } + // One statement, so a round of one, run without a transaction, is atomic var err error if q.priority { - _, err = insert.Exec(data, now, now, priority) + _, err = insert.Exec(data, now, now, jobID, priority) } else { - _, err = insert.Exec(data, now, now) + _, err = insert.Exec(data, now, now, jobID) } + // A repeat id changes no row and is no error: it succeeds return err == nil, err }) } From 6a5b193100119d411a5cd8b068c65a91ddfdce97 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 16:54:48 +0600 Subject: [PATCH 2/3] docs: EnqueueOnce, and varmq v1.10.0 for batch pops and EnqueueOnce A short EnqueueOnce section: a repeat stores nothing and returns true, and the id is remembered while its row exists (pending, processing, or completed with WithRemoveOnComplete(false)). The Compatibility line now says batch pops and EnqueueOnce need varmq v1.10.0 or later, and the schema list names job_id. --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ff8415..4ec94c6 100644 --- a/README.md +++ b/README.md @@ -168,9 +168,20 @@ In the `Mixed1Q` and `Drain1Q` rows of the [varmq-benchmarks `sqlite` benchmark] The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. +## EnqueueOnce + +`EnqueueOnce(item, id)` on a queue, and `EnqueueOnce(item, priority, id)` on a priority queue, add an item under an id unless a row with that id is already stored. A repeat stores nothing and returns true, so an add that is retried with the same id is stored once. + +```go +queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // stored +queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // true, nothing stored +``` + +The id is remembered while its row exists: pending, processing, or completed with `WithRemoveOnComplete(false)`. `Dequeue` deletes the row, and so does the ack with the default remove-on-complete; after that the id can be added again. `Purge` forgets every id. An empty id behaves like `Enqueue`. + ## Compatibility -sqliteq does not import varmq. Queues and priority queues implement `DequeueBatchWithAckIds`, so with varmq ≥ v1.10.0 a worker claims several jobs in one statement. An older varmq never calls it and claims one job per call, which still works. `EnqueueOnce` comes in a later release. +sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker claims several jobs in one statement through `DequeueBatchWithAckIds`, and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. ## How It Works @@ -180,6 +191,7 @@ SQLiteQ uses a SQLite database to store queue items with the following schema: - `data`: The serialized item data (stored as a JSON blob) - `status`: The status of the item ("pending", "processing", or "completed") - `ack_id`: Kept from v0.2 so old tables need no rebuild; no longer used +- `job_id`: The id an item was added with through `EnqueueOnce`, unique among the rows that have one; NULL for `Enqueue`. Opening an older table adds it - `created_at`: When the item was added to the queue - `updated_at`: When the item was last updated From e8c067e0a91704a6e6823de445b2a007080ee980 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 17:36:30 +0600 Subject: [PATCH 3/3] fix: reject nil options, pin EnqueueOnce across recovery, README wording A nil option panicked in New, in the registry's option loop and in newQueue. All three now return one error, so New, NewQueue and NewPriorityQueue never panic on it; a rejected call creates no folder, file or table. Two tests pin the EnqueueOnce id: a job claimed and never acked keeps its id through the requeue on the next open, so a repeat stores nothing and the job runs once; Purge frees the ids of pending and processing rows. README: any name SQLite accepts as a table name under WithSingleFile, the same handle only while it is open, the single-file layout does not create its folder, batch claims only for a worker whose sole source is the sqliteq queue, a Durability section for WAL with synchronous=NORMAL, and the Windows device names as queue names in the folder layout. --- README.md | 14 ++++++--- dedup_test.go | 80 +++++++++++++++++++++++++++++++++++++++++++++++++- layout_test.go | 61 ++++++++++++++++++++++++++++++++++++++ option.go | 6 ++++ queue.go | 3 ++ queues.go | 3 ++ registry.go | 3 ++ 7 files changed, 165 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4ec94c6..e82a0d8 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,9 @@ func main() { } ``` -`New` returns an error instead of panicking: for a path it cannot use, an invalid option, or a path that is an existing file when `WithSingleFile()` is not given. +`New` returns an error instead of panicking: for a path it cannot use, an invalid option (`WithCheckpointPages` below 1, or a nil option), or a path that is an existing file when `WithSingleFile()` is not given. `NewQueue` and `NewPriorityQueue` return an error for a nil option too. -The same name always returns the same queue handle, in any letter case. Asking for `NewQueue("x")` after `NewPriorityQueue("x")` (or the reverse) is an error, and so is reaching a live queue again with different options. `Queue.Close` stops further `Enqueue` and `Dequeue` calls on that handle but still accepts `Acknowledge`, so jobs that are still running can finish. The file stays open until `Queues.Close`. +While a handle is open, the same name returns the same queue handle, in any letter case. Asking for `NewQueue("x")` after `NewPriorityQueue("x")` (or the reverse) is an error, and so is reaching a live queue again with different options. `Queue.Close` stops further `Enqueue` and `Dequeue` calls on that handle but still accepts `Acknowledge`, so jobs that are still running can finish. The file stays open until `Queues.Close`. ## Layout on disk @@ -138,11 +138,13 @@ if err != nil { defer queuesManager.Close() ``` +The file is created if it is missing, but its folder must already exist: `New` returns an error otherwise, unlike the folder layout, which creates its folder. + If you upgrade from v0.2, add `WithSingleFile()` to keep reading your existing file, or start with the folder layout for new data. ### Queue names -In the folder layout the queue name becomes a file name, so it may only contain `A-Z a-z 0-9 . _ -`, must not be empty, and must not start with `sqlite_` (SQLite reserves that prefix). Anything else is an error from `NewQueue` / `NewPriorityQueue`. With `WithSingleFile()` any name works, since it is only a table name. +In the folder layout the queue name becomes a file name, so it may only contain `A-Z a-z 0-9 . _ -`, must not be empty, and must not start with `sqlite_` (SQLite reserves that prefix). Anything else is an error from `NewQueue` / `NewPriorityQueue`. On Windows the folder layout also cannot use a reserved device name as a queue name (`CON`, `PRN`, `AUX`, `NUL`, `COM1` to `COM9`, `LPT1` to `LPT9`, in any letter case), because the name would become a file name such as `con.db`. sqliteq does not check for these. With `WithSingleFile()` the name is only a table name, so any name SQLite accepts as a table name works; SQLite reserves names that start with `sqlite_`. Names are case-insensitive in both layouts: `Jobs` and `jobs` are the same queue, stored in `jobs.db` in the folder layout. @@ -168,6 +170,10 @@ In the `Mixed1Q` and `Drain1Q` rows of the [varmq-benchmarks `sqlite` benchmark] The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. +## Durability + +sqliteq runs SQLite in WAL mode with `synchronous=NORMAL`, unchanged from v0.2. A job whose `Enqueue` returned true survives a crash of the process. After a power loss or an operating system crash, the last commits before the failure may be lost, because SQLite rolls back what the WAL had not yet synced; the database itself stays consistent. + ## EnqueueOnce `EnqueueOnce(item, id)` on a queue, and `EnqueueOnce(item, priority, id)` on a priority queue, add an item under an id unless a row with that id is already stored. A repeat stores nothing and returns true, so an add that is retried with the same id is stored once. @@ -181,7 +187,7 @@ The id is remembered while its row exists: pending, processing, or completed wit ## Compatibility -sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker claims several jobs in one statement through `DequeueBatchWithAckIds`, and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. +sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker whose only source is the sqliteq queue claims several jobs in one statement through `DequeueBatchWithAckIds` (a worker bound to several queues claims one job per call), and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. ## How It Works diff --git a/dedup_test.go b/dedup_test.go index b2aee68..35707c2 100644 --- a/dedup_test.go +++ b/dedup_test.go @@ -21,7 +21,13 @@ type once func(item any, id string) bool // and returns its Queue with its EnqueueOnce. countRows reads table "q". func openOnce(t *testing.T, priority bool, opts ...Option) (*Queue, once, Queues) { t.Helper() - qs := mustNew(t, t.TempDir()) + return openOnceAt(t, t.TempDir(), priority, opts...) +} + +// openOnceAt is openOnce in the folder store at dir, so a test can reopen it +func openOnceAt(t *testing.T, dir string, priority bool, opts ...Option) (*Queue, once, Queues) { + t.Helper() + qs := mustNew(t, dir) if priority { pq, err := qs.NewPriorityQueue("q", opts...) if err != nil { @@ -146,10 +152,82 @@ func testOnceAfterAck(t *testing.T, priority bool) { } } +// A job claimed and never acked is requeued on the next open, and its id +// still blocks a repeat: the job is stored once and runs once. +func TestEnqueueOnceAcrossRecovery(t *testing.T) { testOnceAcrossRecovery(t, false) } + +func testOnceAcrossRecovery(t *testing.T, priority bool) { + dir := t.TempDir() + q, add, qs := openOnceAt(t, dir, priority) + if !add([]byte("a"), "j1") { + qs.Close() + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { + qs.Close() + t.Fatal("DequeueWithAckId failed") + } + // The process "dies": the file closes with the row processing and no ack + if err := qs.Close(); err != nil { + t.Fatalf("Failed to close: %v", err) + } + + q, add, qs = openOnceAt(t, dir, priority) + defer qs.Close() + if n := q.Len(); n != 1 { + t.Fatalf("Len() = %d after reopen, want the claimed job back (1)", n) + } + + if !add([]byte("b"), "j1") { + t.Error("A repeat of the recovered job returned false, want true") + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row, got %d", n) + } + if n := countRows(t, q.f.db, "job_id = 'j1'"); n != 1 { + t.Errorf("Expected 1 row for j1, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[a]" { + t.Errorf("Dequeued %v, want [a] once", got) + } +} + +// Purge deletes the rows, so it frees their ids, pending and processing alike +func TestEnqueueOnceAfterPurge(t *testing.T) { testOnceAfterPurge(t, false) } + +func testOnceAfterPurge(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") || !add([]byte("b"), "j2") { + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { // j1 is processing, j2 pending + t.Fatal("DequeueWithAckId failed") + } + + q.Purge() + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Fatalf("Expected no rows after Purge, got %d", n) + } + + if !add([]byte("c"), "j1") || !add([]byte("d"), "j2") { + t.Error("EnqueueOnce of a purged id returned false, want true") + } + if n := countRows(t, q.f.db, "1"); n != 2 { + t.Errorf("Expected 2 new rows, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c d]" { + t.Errorf("Dequeued %v, want [c d]", got) + } +} + func TestEnqueueOncePriority(t *testing.T) { t.Run("Repeat", func(t *testing.T) { testOnceRepeat(t, true) }) t.Run("WhileProcessing", func(t *testing.T) { testOnceWhileProcessing(t, true) }) t.Run("AfterAck", func(t *testing.T) { testOnceAfterAck(t, true) }) + t.Run("AcrossRecovery", func(t *testing.T) { testOnceAcrossRecovery(t, true) }) + t.Run("AfterPurge", func(t *testing.T) { testOnceAfterPurge(t, true) }) t.Run("Order", func(t *testing.T) { qs := mustNew(t, t.TempDir()) diff --git a/layout_test.go b/layout_test.go index 0ff30a7..1b3bd0e 100644 --- a/layout_test.go +++ b/layout_test.go @@ -3,6 +3,7 @@ package sqliteq import ( "context" "database/sql" + "errors" "fmt" "os" "path/filepath" @@ -129,6 +130,66 @@ func TestNewBadPath(t *testing.T) { } } +// A nil option is an error from New, NewQueue and NewPriorityQueue, not a +// panic, and a rejected call leaves no folder, file or table behind +func TestNilOption(t *testing.T) { + t.Run("New", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "data") + for _, opts := range [][]StoreOption{{nil}, {WithSingleFile(), nil}} { + qs, err := New(dir, opts...) + if err == nil { + qs.Close() + t.Fatalf("Expected an error for a nil StoreOption in %d options", len(opts)) + } + if !errors.Is(err, errNilOption) { + t.Errorf("Got %v, want the nil option error", err) + } + } + if _, err := os.Stat(dir); err == nil { + t.Error("New created the folder before it rejected the option") + } + }) + + t.Run("NewQueue", func(t *testing.T) { + dir := t.TempDir() + qs := mustNew(t, dir) + defer qs.Close() + + if _, err := qs.NewQueue("q", nil); !errors.Is(err, errNilOption) { + t.Errorf("NewQueue with a nil option: got %v, want the nil option error", err) + } + if _, err := qs.NewPriorityQueue("p", nil); !errors.Is(err, errNilOption) { + t.Errorf("NewPriorityQueue with a nil option: got %v, want the nil option error", err) + } + if got := dbFiles(t, dir); len(got) != 0 { + t.Errorf("Expected no database file after the rejected calls, got %v", got) + } + + // The names are still free + if _, err := qs.NewQueue("q"); err != nil { + t.Errorf("Failed to create queue after the rejected call: %v", err) + } + if _, err := qs.NewPriorityQueue("p"); err != nil { + t.Errorf("Failed to create priority queue after the rejected call: %v", err) + } + }) + + // The registry rejects it before newQueue runs; newQueue's own check + // is the second line, and it must stop before it creates the table + t.Run("newQueue", func(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + if _, err := newQueue(q.f, "other", false, nil); !errors.Is(err, errNilOption) { + t.Errorf("newQueue with a nil option: got %v, want the nil option error", err) + } + var n int + if err := q.f.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name = 'other'`).Scan(&n); err != nil || n != 0 { + t.Errorf("Expected no table \"other\", got count %d, err %v", n, err) + } + }) +} + // dbFiles lists the database files in dir, leaving out SQLite's -wal and -shm files. func dbFiles(t *testing.T, dir string) []string { t.Helper() diff --git a/option.go b/option.go index ad485f6..c56379f 100644 --- a/option.go +++ b/option.go @@ -1,5 +1,11 @@ package sqliteq +import "errors" + +// errNilOption is returned by New, NewQueue and NewPriorityQueue for a nil +// option, which they would otherwise call +var errNilOption = errors.New("sqliteq: option must not be nil") + // Option is a function type that can be used to configure a Queue type Option func(*Queue) diff --git a/queue.go b/queue.go index c2f62a3..512555e 100644 --- a/queue.go +++ b/queue.go @@ -55,6 +55,9 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro // Apply any provided options for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(q) } diff --git a/queues.go b/queues.go index 490f63e..14836db 100644 --- a/queues.go +++ b/queues.go @@ -29,6 +29,9 @@ type Queues interface { func New(path string, opts ...StoreOption) (Queues, error) { cfg := defaultStoreConfig() for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(&cfg) } diff --git a/registry.go b/registry.go index 74d6151..24aeb0e 100644 --- a/registry.go +++ b/registry.go @@ -136,6 +136,9 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) want := Queue{removeOnComplete: true} for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(&want) }