diff --git a/CHANGELOG.md b/CHANGELOG.md index a0d0165..2b5eabe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to the SQLiteQ project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is not stored. + ## [0.3.0] - 2026-10-23 ### Breaking diff --git a/README.md b/README.md index 6b4c501..d911546 100644 --- a/README.md +++ b/README.md @@ -122,14 +122,19 @@ data, ok := pq.Dequeue() // "password reset" ```go item, ok, ackID := queue.DequeueWithAckId() if ok { - process(item) - queue.Acknowledge(ackID) + if err := process(item); err != nil { + queue.Reject(ackID, err.Error()) // failed for good: not requeued + } else { + queue.Acknowledge(ackID) + } } ``` An item taken with `DequeueWithAckId` is marked as processing until it is acknowledged. The first time a queue is opened after its file is opened in the process, every item still marked as processing goes back to pending, so a job that was running when the process stopped runs again. There is no visibility timeout: unacknowledged items return only when the file is next opened, not when a closed queue is opened again on a file that is still open. -By default an acknowledged item is deleted. With `WithRemoveOnComplete(false)` it is kept and marked as completed: +`Reject(ackID, reason)` finishes an item that failed for good, as varmq does when a job returns an error or panics. It is never requeued. The reason is not stored. + +By default an acknowledged or rejected item is deleted. With `WithRemoveOnComplete(false)` it is kept and marked as `completed` or `failed`: ```go audit, err := qs.NewQueue("audit", sqliteq.WithRemoveOnComplete(false)) @@ -146,7 +151,7 @@ queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // true, nothing stor pq.EnqueueOnce([]byte("rebuild index"), 1, "rebuild-index") ``` -The id is remembered while its row exists: pending, processing, or completed with `WithRemoveOnComplete(false)`. `Dequeue` deletes the row, and so does the ack with the default remove-on-complete; after that the id can be added again. `Purge` forgets every id. An empty id behaves like `Enqueue`. +The id is remembered while its row exists: pending, processing, or completed or failed with `WithRemoveOnComplete(false)`. `Dequeue` deletes the row, and so does the ack or reject with the default remove-on-complete; after that the id can be added again. `Purge` forgets every id. An empty id behaves like `Enqueue`. ### Batch dequeue @@ -237,7 +242,7 @@ On macOS, spell a path the same way every time: the registry key keeps the lette | ----------------------------- | ------------------------------ | ------------- | ---------------------------------------------------------------------- | | `WithSingleFile()` | `New` | folder layout | `path` is one database file holding every queue | | `WithCheckpointPages(n)` | `New` | `1000` | WAL size, in pages, at which SQLite checkpoints automatically | -| `WithRemoveOnComplete(false)` | `NewQueue`, `NewPriorityQueue` | `true` | Keep acknowledged items, marked as completed, instead of deleting them | +| `WithRemoveOnComplete(false)` | `NewQueue`, `NewPriorityQueue` | `true` | Keep acknowledged and rejected items, marked as `completed` or `failed`, instead of deleting them | `New` returns an error instead of panicking: for a path it cannot use, an invalid option (`WithCheckpointPages` below 1, or a nil option), or a path that is an existing file when `WithSingleFile()` is not given. `NewQueue` and `NewPriorityQueue` return an error for a nil option too. @@ -286,7 +291,7 @@ Each queue is a table with these columns: - `priority`: priority queues only; a lower number comes out first. Opening an older table as a priority queue adds it. - `data`: the stored item. - `status`: `pending`, `processing` or `completed`. -- `ack`: 0 when an item is added, 1 once it is acknowledged and kept with `WithRemoveOnComplete(false)`. On open, only `processing` rows with `ack` 0 go back to pending. +- `ack`: 0 when an item is added, 1 once it is acknowledged or rejected and kept with `WithRemoveOnComplete(false)` (`status` `completed` or `failed`). On open, only `processing` rows with `ack` 0 go back to pending. - `ack_id`: kept from v0.2 so old tables need no rebuild; no longer used. - `job_id`: the id an item was added with through `EnqueueOnce`, unique among the rows that have one; NULL for `Enqueue`. Opening an older table adds it. - `created_at`, `updated_at`: when the item was added and last changed. diff --git a/queue.go b/queue.go index 512555e..7d41be6 100644 --- a/queue.go +++ b/queue.go @@ -27,7 +27,7 @@ type Queue struct { closed atomic.Bool // Prepared once per handle; they close with the file's pool - insert, claim, pop, ack, count, countCapped, values *sql.Stmt + insert, claim, pop, ack, reject, count, countCapped, values *sql.Stmt } // claimed is one row a claim moved to processing @@ -205,8 +205,11 @@ func (q *Queue) prepare() (err error) { pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order) ack := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing'", q.table) + reject := ack if !q.removeOnComplete { - ack = fmt.Sprintf("UPDATE %s SET status = 'completed', ack = 1, updated_at = ? WHERE id = ? AND status = 'processing'", q.table) + finish := "UPDATE %s SET status = '%s', ack = 1, updated_at = ? WHERE id = ? AND status = 'processing'" + ack = fmt.Sprintf(finish, q.table, "completed") + reject = fmt.Sprintf(finish, q.table, "failed") } for _, s := range []struct { @@ -219,6 +222,7 @@ func (q *Queue) prepare() (err error) { {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)}, {&q.ack, ack}, + {&q.reject, reject}, {&q.count, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", q.table)}, {&q.countCapped, fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE status = 'pending' LIMIT %d)", q.table, remainingCap)}, {&q.values, fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)}, @@ -233,7 +237,7 @@ func (q *Queue) prepare() (err error) { // closeStmts closes the statements prepared so far func (q *Queue) closeStmts() { - for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.ack, q.count, q.countCapped, q.values} { + for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.ack, q.reject, q.count, q.countCapped, q.values} { if s != nil { s.Close() } @@ -467,7 +471,16 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini // Acknowledge marks a processing item as completed, deleting it unless // WithRemoveOnComplete(false) is set, in the file's next group commit round // Returns true only if exactly one processing row had this ack ID -func (q *Queue) Acknowledge(ackID string) bool { +func (q *Queue) Acknowledge(ackID string) bool { return q.finish(ackID, q.ack) } + +// Reject marks a processing item as failed for good: it is not requeued. +// The row is deleted, or with WithRemoveOnComplete(false) kept with status +// 'failed'. Same group commit and result as Acknowledge. +// ponytail: reason is not stored (no column); add one when something reads it. +func (q *Queue) Reject(ackID string, reason string) bool { return q.finish(ackID, q.reject) } + +// finish runs stmt (ack or reject) for the row of ackID. +func (q *Queue) finish(ackID string, stmt *sql.Stmt) bool { id, ok := parseAckID(ackID) if !ok { return false @@ -479,14 +492,14 @@ func (q *Queue) Acknowledge(ackID string) bool { result sql.Result err error ) - ack := q.ack + s := stmt if tx != nil { - ack = tx.Stmt(ack) + s = tx.Stmt(s) } if q.removeOnComplete { - result, err = ack.Exec(id) + result, err = s.Exec(id) } else { - result, err = ack.Exec(now, id) + result, err = s.Exec(now, id) } if err != nil { return false, err diff --git a/reject_test.go b/reject_test.go new file mode 100644 index 0000000..2f2b794 --- /dev/null +++ b/reject_test.go @@ -0,0 +1,82 @@ +package sqliteq + +import "testing" + +// Reject removes a processing row for good: it is not requeued, a second +// Reject or an Acknowledge of the same ack ID fails, and a bad ack ID fails. +func TestReject(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + q.Enqueue("bad job") + q.Enqueue("next") + _, ok, ackID := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId failed") + } + + if !q.Reject(ackID, "boom") { + t.Fatal("Reject failed") + } + if n := countRows(t, q.f.db, "1 = 1"); n != 1 { + t.Errorf("Expected 1 row left (the pending one), got %d", n) + } + if err := q.requeueNoAck(); err != nil { // what a restart runs + t.Fatal(err) + } + if n := q.Len(); n != 1 { + t.Errorf("Expected Len 1 after a requeue sweep (rejected job not back), got %d", n) + } + if q.Reject(ackID, "again") || q.Acknowledge(ackID) { + t.Error("Expected a second finish of the same ack ID to fail") + } + if q.Reject("not-an-ack-id", "x") { + t.Error("Expected Reject of a malformed ack ID to fail") + } +} + +// With WithRemoveOnComplete(false) a rejected row is kept as 'failed', +// apart from acknowledged rows kept as 'completed'. +func TestRejectKeepsFailedRow(t *testing.T) { + q, qs := openQueue(t, WithRemoveOnComplete(false)) + defer qs.Close() + + q.Enqueue("ok") + q.Enqueue("bad") + _, _, okID := q.DequeueWithAckId() + _, _, badID := q.DequeueWithAckId() + if !q.Acknowledge(okID) || !q.Reject(badID, "boom") { + t.Fatal("finish failed") + } + + if n := countRows(t, q.f.db, "status = 'completed' AND ack = 1"); n != 1 { + t.Errorf("Expected 1 completed row, got %d", n) + } + if n := countRows(t, q.f.db, "status = 'failed' AND ack = 1"); n != 1 { + t.Errorf("Expected 1 failed row, got %d", n) + } + if err := q.requeueNoAck(); err != nil { + t.Fatal(err) + } + if n := q.Len(); n != 0 { + t.Errorf("Expected Len 0 (finished rows never requeue), got %d", n) + } +} + +// A priority queue rejects through the embedded Queue. +func TestPriorityReject(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("p") + if err != nil { + t.Fatal(err) + } + pq.Enqueue("x", 1) + _, ok, ackID := pq.DequeueWithAckId() + if !ok || !pq.Reject(ackID, "boom") { + t.Fatal("priority Reject failed") + } + if pq.Len() != 0 { + t.Errorf("Expected empty priority queue, got %d", pq.Len()) + } +}