From 35cf8ade05fcef9fd437d042bb6481d810c4c613 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 21:30:05 +0600
Subject: [PATCH 01/22] build: pin varmq and contract at 0a9288e
Go 1.27 (varmq root needs it). contract pinned too: varmq's replace
is not inherited.
---
go.mod | 8 ++++++--
go.sum | 12 ++++++++++++
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/go.mod b/go.mod
index b6ff707..6b04aeb 100644
--- a/go.mod
+++ b/go.mod
@@ -1,5 +1,9 @@
module github.com/goptics/sqliteq
-go 1.24
+go 1.27
-require github.com/mattn/go-sqlite3 v1.14.28
+require (
+ github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092
+ github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092
+ github.com/mattn/go-sqlite3 v1.14.28
+)
diff --git a/go.sum b/go.sum
index 42e5bac..f4edda0 100644
--- a/go.sum
+++ b/go.sum
@@ -1,2 +1,14 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092 h1:oFPxzb1mXotIzHHKlxz+kUGO1ASrWZkg2nmkYhrgx60=
+github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E=
+github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092 h1:wWq1t3n0//1/URO+uwwdEybf6GJcVHmmtJ9lcKAHxRY=
+github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg=
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
From 010dd7c472ad48b21c1e8490954648aeffdbf370 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 21:30:05 +0600
Subject: [PATCH 02/22] feat: job outcome and cancel for varmq
OutcomeWatcher, ResultAcknowledgeable, JobCanceller, CancelNotifier,
CancelAcknowledgeable on Queue and PriorityQueue (spec 6.13, 6.14).
Sidecar tables __sqliteq_jobs (outcome per Job ID, retention)
and __sqliteq_cancelled (marks, AUTOINCREMENT seq). Job ID =
job_id, else item JSON id. Finish writes outcome + clears mark in its
tx. Pops gate on live marks, drop with outcome cancelled, cap 1000.
EnqueueOnce re-add clears both via AFTER INSERT trigger. Purge clears
both. No pub/sub: one poll goroutine per queue object, 50 ms, only
while watches or hooks exist. Txs begin IMMEDIATE.
---
cancel.go | 172 ++++++++++++++++
outcome.go | 326 ++++++++++++++++++++++++++++++
priority_queue_test.go | 44 ++++
queue.go | 245 ++++++++++++++++------
queue_test.go | 446 +++++++++++++++++++++++++++++++++++++++++
registry.go | 9 +-
6 files changed, 1181 insertions(+), 61 deletions(-)
create mode 100644 cancel.go
create mode 100644 outcome.go
diff --git a/cancel.go b/cancel.go
new file mode 100644
index 0000000..ce8f1c9
--- /dev/null
+++ b/cancel.go
@@ -0,0 +1,172 @@
+package sqliteq
+
+import (
+ "database/sql"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/goptics/varmq/contract"
+)
+
+// Job cancel (varmq spec/redis-storage.md 1.7.0, 6.14), on SQLite.
+// CancelJob writes a mark to __sqliteq_cancelled (expires_at: ms).
+// Every pop drops a marked waiting job with outcome cancelled; OnCancel
+// learns of marks by polling seq; AcknowledgeCancelled finishes a running
+// job that stopped for it. Any finish, and an EnqueueOnce that stores the
+// id, clears the mark. sqliteq has no workflow store, so no item is exempt.
+
+var (
+ _ contract.JobCanceller = (*Queue)(nil)
+ _ contract.JobCanceller = (*PriorityQueue)(nil)
+ _ contract.CancelNotifier = (*Queue)(nil)
+ _ contract.CancelNotifier = (*PriorityQueue)(nil)
+ _ contract.CancelAcknowledgeable = (*Queue)(nil)
+ _ contract.CancelAcknowledgeable = (*PriorityQueue)(nil)
+)
+
+const (
+ // cancelTTL is how long one mark lasts.
+ cancelTTL = 24 * time.Hour
+ // maxDrops caps the marked items one pop drops; at the cap it returns
+ // what it has and the next pop goes on.
+ maxDrops = 1000
+)
+
+var (
+ errEmptyJobID = errors.New("sqliteq: empty job id")
+ errClosed = errors.New("sqliteq: queue is closed")
+)
+
+// CancelJob implements contract.JobCanceller: it marks job id cancelled for
+// cancelTTL. A waiting job is dropped by the pop that would hand it out
+// (outcome cancelled); OnCancel callbacks of every queue object on the file
+// learn of a running one, which ends only if its worker stops it.
+// contract.ErrJobEnded when its outcome record shows it ended. An unknown
+// id is marked too: a later Enqueue of it is dropped, unless EnqueueOnce
+// stores it (that clears the mark).
+func (q *Queue) CancelJob(id string) error {
+ if id == "" {
+ return errEmptyJobID
+ }
+ if q.closed.Load() {
+ return errClosed
+ }
+
+ now := time.Now().UnixMilli()
+ q.f.mu.Lock()
+ defer q.f.mu.Unlock()
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ var state string
+ switch err := tx.QueryRow(fmt.Sprintf("SELECT state FROM %s WHERE job_id = ?", q.jobs), id).Scan(&state); {
+ case err == nil:
+ return contract.ErrJobEnded // every record is an end
+ case !errors.Is(err, sql.ErrNoRows):
+ return err
+ }
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE expires_at <= ?", q.cancelled), now); err != nil {
+ return err
+ }
+ // REPLACE deletes an older mark of id, so the new one takes a new seq
+ // and reaches OnCancel again
+ if _, err := tx.Exec(fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, expires_at) VALUES (?, ?)", q.cancelled),
+ id, now+cancelTTL.Milliseconds()); err != nil {
+ return err
+ }
+ return tx.Commit()
+}
+
+// AcknowledgeCancelled implements contract.CancelAcknowledgeable: Reject
+// with outcome cancelled and reason as its error. A kept row
+// (WithRemoveOnComplete(false)) gets status 'cancelled'.
+func (q *Queue) AcknowledgeCancelled(ackID, reason string) bool {
+ return q.finish(ackID, contract.JobCancelled, reason, nil)
+}
+
+// OnCancel implements contract.CancelNotifier: fn gets the job ID of each
+// cancel request made after the call, through any queue object on this
+// database file, until stop. Marks are polled every pollEvery; each call
+// runs on its own goroutine. A mark gone by the poll (the job finished or
+// was re-added) is never reported. A closed queue reports nothing.
+func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) {
+ // Read before the lock: a mark made after this read has a higher seq
+ var last sql.NullInt64
+ _ = q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.cancelled)).Scan(&last)
+
+ p := &fn
+ w := &q.w
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.closed {
+ return func() {}
+ }
+ if len(w.hooks) == 0 {
+ w.lastSeq = last.Int64
+ }
+ if w.hooks == nil {
+ w.hooks = map[*func(string)]struct{}{}
+ }
+ w.hooks[p] = struct{}{}
+ w.startLocked(q)
+
+ return func() {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ delete(w.hooks, p)
+ }
+}
+
+// pollCancels passes the live marks newer than seq to hooks.
+func (q *Queue) pollCancels(seq int64, hooks []func(string)) {
+ rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? AND expires_at > ? ORDER BY seq LIMIT %d",
+ q.cancelled, maxDrops), seq, time.Now().UnixMilli())
+ if err != nil {
+ return
+ }
+ var ids []string
+ for rows.Next() {
+ var id string
+ if rows.Scan(&seq, &id) != nil {
+ continue
+ }
+ ids = append(ids, id)
+ }
+ if rows.Close() != nil || rows.Err() != nil {
+ return // read again from the old seq
+ }
+
+ w := &q.w
+ w.mu.Lock()
+ w.lastSeq = max(w.lastSeq, seq)
+ w.mu.Unlock()
+ for _, id := range ids {
+ for _, fn := range hooks {
+ go fn(id)
+ }
+ }
+}
+
+// anyMark reports whether a live mark exists, so a pop without marks pays
+// one read; an error counts as a mark (the gated pop then reports it).
+func (q *Queue) anyMark(now time.Time) bool {
+ var n int
+ if err := q.hasMark.QueryRow(now.UnixMilli()).Scan(&n); err != nil {
+ return true
+ }
+ return n == 1
+}
+
+// marked reports whether job id has a live mark, in tx.
+func (q *Queue) marked(tx *sql.Tx, id string, now time.Time) (bool, error) {
+ if id == "" {
+ return false, nil
+ }
+ var n int
+ err := tx.Stmt(q.isMarked).QueryRow(id, now.UnixMilli()).Scan(&n)
+ return n == 1, err
+}
diff --git a/outcome.go b/outcome.go
new file mode 100644
index 0000000..5aa91e4
--- /dev/null
+++ b/outcome.go
@@ -0,0 +1,326 @@
+package sqliteq
+
+import (
+ "database/sql"
+ "encoding/json"
+ "fmt"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/goptics/varmq/contract"
+)
+
+// Job outcome (varmq spec/redis-storage.md 1.6.0, 6.13), on SQLite. Each
+// finish of a job with a Job ID writes one row to __sqliteq_jobs in
+// the finish's transaction. SQLite has no pub/sub, so a watcher is
+// answered by polling that table: one goroutine per queue object, alive
+// only while it has watches or OnCancel hooks.
+
+var (
+ _ contract.OutcomeWatcher = (*Queue)(nil)
+ _ contract.OutcomeWatcher = (*PriorityQueue)(nil)
+ _ contract.ResultAcknowledgeable = (*Queue)(nil)
+ _ contract.ResultAcknowledgeable = (*PriorityQueue)(nil)
+)
+
+const (
+ // pollEvery is how often a queue object with watches or hooks reads
+ // the outcome and cancel tables.
+ pollEvery = 50 * time.Millisecond
+ // maxCatchUp caps the watched ids one poll reads; map order rotates a
+ // larger set.
+ maxCatchUp = 500
+
+ // Retention (spec 5 defaults): cancelled records count as failed.
+ keepCompleted = 1000
+ keepCompletedAge = 24 * time.Hour
+ keepFailed = 10000
+ keepFailedAge = 7 * 24 * time.Hour
+ // trimEvery is how many records a queue object writes between trims.
+ trimEvery = 100
+
+ jobsSuffix = "__sqliteq_jobs"
+ cancelledSuffix = "__sqliteq_cancelled"
+ readdSuffix = "__sqliteq_readd"
+)
+
+// initSidecars creates the outcome and cancel tables, and the trigger that
+// clears both for an id EnqueueOnce stores (spec 6.13 and 6.14 Re-add): an
+// ON CONFLICT DO NOTHING insert of a repeat id fires no trigger. The caller
+// holds the write mutex.
+func (q *Queue) initSidecars() error {
+ for _, s := range []string{
+ `CREATE TABLE IF NOT EXISTS %[2]s (
+ job_id TEXT PRIMARY KEY,
+ state TEXT NOT NULL,
+ error TEXT,
+ result BLOB,
+ finished_at INTEGER NOT NULL
+ ) WITHOUT ROWID`,
+ `CREATE INDEX IF NOT EXISTS %[4]s ON %[2]s (finished_at)`,
+ // AUTOINCREMENT: seq never reuses a deleted mark's, so OnCancel's
+ // seq > last cursor sees every new mark
+ `CREATE TABLE IF NOT EXISTS %[3]s (
+ seq INTEGER PRIMARY KEY AUTOINCREMENT,
+ job_id TEXT NOT NULL UNIQUE,
+ expires_at INTEGER NOT NULL
+ )`,
+ `CREATE TRIGGER IF NOT EXISTS %[5]s AFTER INSERT ON %[1]s WHEN NEW.job_id IS NOT NULL BEGIN
+ DELETE FROM %[2]s WHERE job_id = NEW.job_id;
+ DELETE FROM %[3]s WHERE job_id = NEW.job_id;
+ END`,
+ } {
+ query := fmt.Sprintf(s, q.table, q.jobs, q.cancelled,
+ quoteIdent(q.name+jobsSuffix+"_finished"), quoteIdent(q.name+readdSuffix))
+ if _, err := q.f.db.Exec(query); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// jobIDOf is a row's Job ID: its EnqueueOnce id, else the envelope "id" of
+// a JSON object item (varmq's generated ids come with plain Enqueue). ""
+// when it has neither: such a job has no outcome and is never cancelled.
+func jobIDOf(onceID sql.NullString, data []byte) string {
+ if onceID.Valid {
+ return onceID.String
+ }
+ if len(data) == 0 || data[0] != '{' {
+ return ""
+ }
+ var env struct {
+ ID string `json:"id"`
+ }
+ if json.Unmarshal(data, &env) != nil {
+ return ""
+ }
+ return env.ID
+}
+
+// record writes job id's outcome and drops its cancel mark, in tx; every
+// trimEvery-th record also trims both states.
+func (q *Queue) record(tx *sql.Tx, id string, state contract.JobState, reason string, result []byte, now time.Time) error {
+ var errText any
+ if state != contract.JobCompleted {
+ errText = reason
+ }
+ if _, err := tx.Stmt(q.putJob).Exec(id, string(state), errText, result, now.UnixMilli()); err != nil {
+ return err
+ }
+ if _, err := tx.Stmt(q.unmark).Exec(id); err != nil {
+ return err
+ }
+ if q.records.Add(1)%trimEvery == 0 {
+ return q.trim(tx, now)
+ }
+ return nil
+}
+
+// trim applies retention: per state group, records past the age limit and
+// those past the newest N.
+func (q *Queue) trim(tx *sql.Tx, now time.Time) error {
+ for _, g := range []struct {
+ where string
+ keep int
+ age time.Duration
+ }{
+ {"state = 'completed'", keepCompleted, keepCompletedAge},
+ {"state <> 'completed'", keepFailed, keepFailedAge},
+ } {
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE %s AND finished_at <= ?", q.jobs, g.where),
+ now.Add(-g.age).UnixMilli()); err != nil {
+ return err
+ }
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %[1]s WHERE job_id IN (SELECT job_id FROM %[1]s WHERE %[2]s ORDER BY finished_at DESC LIMIT -1 OFFSET %[3]d)",
+ q.jobs, g.where, g.keep)); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// AcknowledgeResult implements contract.ResultAcknowledgeable: Acknowledge
+// that keeps result, which must be valid JSON, in the job's outcome. Not
+// JSON: false, and nothing changes.
+func (q *Queue) AcknowledgeResult(ackID string, result []byte) bool {
+ if !json.Valid(result) {
+ return false
+ }
+ return q.finish(ackID, contract.JobCompleted, "", result)
+}
+
+type outcomeWatch struct {
+ fn func(contract.JobOutcome)
+ done atomic.Bool // called or cancelled
+}
+
+// watchers are a queue object's outcome watches and OnCancel hooks, and the
+// state of its one poll goroutine. mu is never held across a query.
+type watchers struct {
+ mu sync.Mutex
+ byID map[string]map[*outcomeWatch]struct{}
+ hooks map[*func(string)]struct{}
+ lastSeq int64 // newest cancel mark seq OnCancel has seen
+ polling bool
+ closed bool
+ stop chan struct{} // closed by close
+}
+
+// startLocked starts the poll goroutine if none runs; w.mu is held.
+func (w *watchers) startLocked(q *Queue) {
+ if !w.polling {
+ w.polling = true
+ go q.poll()
+ }
+}
+
+// close drops every watch and hook without calling it, and stops polling.
+func (w *watchers) close() {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.closed {
+ return
+ }
+ w.closed = true
+ close(w.stop)
+ for _, set := range w.byID {
+ for o := range set {
+ o.done.Store(true)
+ }
+ }
+ w.byID, w.hooks = nil, nil
+}
+
+// WatchOutcome implements contract.OutcomeWatcher: fn runs once, on its own
+// goroutine, when job id completes, fails or is cancelled through any
+// queue object on this database file, in this process or another. A job
+// whose record is still kept is reported even if it ended before the call.
+// It is polled every pollEvery. An empty id, or a closed queue, watches
+// nothing. Close drops watches without calling fn.
+func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel func()) {
+ o := &outcomeWatch{fn: fn}
+ w := &q.w
+ w.mu.Lock()
+ if id == "" || w.closed {
+ w.mu.Unlock()
+ return func() {}
+ }
+ if w.byID == nil {
+ w.byID = map[string]map[*outcomeWatch]struct{}{}
+ }
+ if w.byID[id] == nil {
+ w.byID[id] = map[*outcomeWatch]struct{}{}
+ }
+ w.byID[id][o] = struct{}{}
+ w.startLocked(q)
+ w.mu.Unlock()
+
+ go q.catchUp([]string{id})
+
+ return func() {
+ if o.done.CompareAndSwap(false, true) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if set := w.byID[id]; set != nil {
+ delete(set, o)
+ if len(set) == 0 {
+ delete(w.byID, id)
+ }
+ }
+ }
+ }
+}
+
+// poll reads the watched outcomes and new cancel marks every pollEvery,
+// until Close or until nothing is watched.
+func (q *Queue) poll() {
+ t := time.NewTicker(pollEvery)
+ defer t.Stop()
+ w := &q.w
+ for {
+ select {
+ case <-w.stop:
+ return
+ case <-t.C:
+ }
+
+ w.mu.Lock()
+ if len(w.byID) == 0 && len(w.hooks) == 0 {
+ w.polling = false
+ w.mu.Unlock()
+ return
+ }
+ ids := make([]string, 0, min(len(w.byID), maxCatchUp))
+ for id := range w.byID {
+ if len(ids) == maxCatchUp {
+ break
+ }
+ ids = append(ids, id)
+ }
+ hooks := make([]func(string), 0, len(w.hooks))
+ for fn := range w.hooks {
+ hooks = append(hooks, *fn)
+ }
+ seq := w.lastSeq
+ w.mu.Unlock()
+
+ if len(ids) > 0 {
+ q.catchUp(ids)
+ }
+ if len(hooks) > 0 {
+ q.pollCancels(seq, hooks)
+ }
+ }
+}
+
+// catchUp delivers the ids whose outcome record exists. A failed read is
+// retried by the next poll.
+func (q *Queue) catchUp(ids []string) {
+ args := make([]any, len(ids))
+ for i, id := range ids {
+ args[i] = id
+ }
+ rows, err := q.f.db.Query(fmt.Sprintf("SELECT job_id, state, error, result FROM %s WHERE job_id IN (%s)",
+ q.jobs, strings.Repeat("?, ", len(ids)-1)+"?"), args...)
+ if err != nil {
+ return
+ }
+ var outs []contract.JobOutcome
+ for rows.Next() {
+ var (
+ id, state string
+ errText sql.NullString
+ result []byte
+ )
+ if rows.Scan(&id, &state, &errText, &result) != nil {
+ continue
+ }
+ outs = append(outs, contract.JobOutcome{ID: id, State: contract.JobState(state), Error: errText.String, Result: result})
+ }
+ rows.Close()
+
+ for _, out := range outs {
+ q.deliver(out)
+ }
+}
+
+// deliver ends every watch of out.ID: each fn runs once, on its own
+// goroutine, so a slow fn holds up no poll.
+func (q *Queue) deliver(out contract.JobOutcome) {
+ w := &q.w
+ w.mu.Lock()
+ set := w.byID[out.ID]
+ delete(w.byID, out.ID)
+ w.mu.Unlock()
+
+ for o := range set {
+ go func() {
+ if o.done.CompareAndSwap(false, true) {
+ o.fn(out)
+ }
+ }()
+ }
+}
diff --git a/priority_queue_test.go b/priority_queue_test.go
index 39b2ed7..a7cadcf 100644
--- a/priority_queue_test.go
+++ b/priority_queue_test.go
@@ -1,10 +1,13 @@
package sqliteq
import (
+ "errors"
"fmt"
"strconv"
"testing"
+ "github.com/goptics/varmq/contract"
+
_ "github.com/mattn/go-sqlite3"
)
@@ -653,3 +656,44 @@ func TestBatchPriorityOrder(t *testing.T) {
}
}
}
+
+// A priority queue reports outcomes, drops marked waiting jobs in priority
+// order, and re-add clears the mark, as Queue does.
+func TestPriorityOutcomeAndCancel(t *testing.T) {
+ qs := mustNew(t, t.TempDir())
+ defer qs.Close()
+ pq, err := qs.NewPriorityQueue("pq")
+ if err != nil {
+ t.Fatal(err)
+ }
+ other := otherProcess(t, pq.Queue)
+
+ pq.Enqueue(job("low"), 5)
+ pq.Enqueue(job("high"), 1)
+ pq.EnqueueOnce(job("mid"), 3, "mid")
+ if err := other.CancelJob("high"); err != nil {
+ t.Fatal(err)
+ }
+ chHigh, chMid := watch(other, "high"), watch(other, "mid")
+
+ if v, ack := pop(t, pq.Queue); v != string(job("mid")) {
+ t.Fatalf("got %s, want mid", v)
+ } else if !pq.AcknowledgeResult(ack, []byte(`7`)) {
+ t.Fatal("AcknowledgeResult failed")
+ }
+ if got := waitOutcome(t, chHigh); got.State != contract.JobCancelled {
+ t.Errorf("high: got %+v", got)
+ }
+ if got := waitOutcome(t, chMid); got.State != contract.JobCompleted || string(got.Result) != "7" {
+ t.Errorf("mid: got %+v", got)
+ }
+ if err := pq.CancelJob("mid"); !errors.Is(err, contract.ErrJobEnded) {
+ t.Errorf("got %v, want ErrJobEnded", err)
+ }
+
+ pq.CancelJob("again")
+ pq.EnqueueOnce(job("again"), 0, "again")
+ if v, _ := pop(t, pq.Queue); v != string(job("again")) {
+ t.Errorf("Expected the re-added job first, got %s", v)
+ }
+}
diff --git a/queue.go b/queue.go
index 7d41be6..471326f 100644
--- a/queue.go
+++ b/queue.go
@@ -4,6 +4,7 @@ import (
"cmp"
"database/sql"
"database/sql/driver"
+ "errors"
"fmt"
"slices"
"strconv"
@@ -11,6 +12,7 @@ import (
"sync/atomic"
"time"
+ "github.com/goptics/varmq/contract"
_ "github.com/mattn/go-sqlite3"
)
@@ -26,8 +28,13 @@ type Queue struct {
removeOnComplete bool
closed atomic.Bool
+ jobs, cancelled string // quoted outcome and cancel mark tables
+ w watchers // outcome watches and OnCancel hooks
+ records atomic.Int64 // outcome records written, for trims
+
// Prepared once per handle; they close with the file's pool
- insert, claim, pop, ack, reject, count, countCapped, values *sql.Stmt
+ insert, claim, pop, fin, count, countCapped, values *sql.Stmt
+ putJob, unmark, hasMark, isMarked *sql.Stmt
}
// claimed is one row a claim moved to processing
@@ -35,6 +42,7 @@ type claimed struct {
id int64
priority int
data []byte
+ once sql.NullString // job_id
}
// newQueue creates a SQLite-based queue on its table: the table itself, the
@@ -48,7 +56,10 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro
priority: priority,
order: "id",
removeOnComplete: true, // Default to removing completed items
+ jobs: quoteIdent(name + jobsSuffix),
+ cancelled: quoteIdent(name + cancelledSuffix),
}
+ q.w.stop = make(chan struct{})
if priority {
q.order = "priority, id"
}
@@ -86,6 +97,9 @@ func (q *Queue) initSchema() error {
if err := q.addJobID(); err != nil {
return fmt.Errorf("failed to add job_id: %w", err)
}
+ if err := q.initSidecars(); err != nil {
+ return fmt.Errorf("failed to create outcome tables: %w", err)
+ }
return nil
}
@@ -198,18 +212,16 @@ func (q *Queue) prepare() (err error) {
}
}()
- cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data"
+ cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data, job_id"
if q.priority {
- cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data"
+ cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data, job_id"
}
pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)
- ack := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing'", q.table)
- reject := ack
+ // fin ends a processing row: deleted, or kept with its end as status
+ fin := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table)
if !q.removeOnComplete {
- finish := "UPDATE %s SET status = '%s', ack = 1, updated_at = ? WHERE id = ? AND status = 'processing'"
- ack = fmt.Sprintf(finish, q.table, "completed")
- reject = fmt.Sprintf(finish, q.table, "failed")
+ fin = fmt.Sprintf("UPDATE %s SET status = ?, ack = 1, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table)
}
for _, s := range []struct {
@@ -220,9 +232,12 @@ func (q *Queue) prepare() (err error) {
// other failure still errors; it must repeat the index's WHERE
{&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING", q.table, cols, vals)},
{&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)},
- {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)},
- {&q.ack, ack},
- {&q.reject, reject},
+ {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data, job_id", q.table, pending)},
+ {&q.fin, fin},
+ {&q.putJob, fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, state, error, result, finished_at) VALUES (?, ?, ?, ?, ?)", q.jobs)},
+ {&q.unmark, fmt.Sprintf("DELETE FROM %s WHERE job_id = ?", q.cancelled)},
+ {&q.hasMark, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE expires_at > ?)", q.cancelled)},
+ {&q.isMarked, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE job_id = ? AND expires_at > ?)", q.cancelled)},
{&q.count, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", q.table)},
{&q.countCapped, fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE status = 'pending' LIMIT %d)", q.table, remainingCap)},
{&q.values, fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)},
@@ -237,7 +252,7 @@ func (q *Queue) prepare() (err error) {
// closeStmts closes the statements prepared so far
func (q *Queue) closeStmts() {
- for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.ack, q.reject, q.count, q.countCapped, q.values} {
+ for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.fin, q.count, q.countCapped, q.values, q.putJob, q.unmark, q.hasMark, q.isMarked} {
if s != nil {
s.Close()
}
@@ -351,7 +366,7 @@ func (q *Queue) enqueue(item any, priority int, id string) bool {
// returns them in queue order
func (q *Queue) claimN(n int) ([]claimed, error) {
q.f.mu.Lock()
- out, err := q.claimRows(n)
+ out, err := q.claimGated(n)
q.f.mu.Unlock()
if err != nil {
return nil, err
@@ -367,10 +382,56 @@ func (q *Queue) claimN(n int) ([]claimed, error) {
return out, nil
}
-// claimRows runs the claim statement and reads every row it returns; the
+// claimGated claims up to n rows; with a live cancel mark on the file it
+// runs in one transaction that drops each marked row it claims (outcome
+// cancelled) and claims again, at most maxDrops drops. The caller holds the
+// write mutex.
+func (q *Queue) claimGated(n int) ([]claimed, error) {
+ now := time.Now()
+ if !q.anyMark(now) {
+ return q.claimRows(q.claim, n)
+ }
+
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return nil, err
+ }
+ defer tx.Rollback()
+
+ claim := tx.Stmt(q.claim)
+ out := make([]claimed, 0, n)
+ for drops := 0; len(out) < n && drops < maxDrops; {
+ rows, err := q.claimRows(claim, n-len(out))
+ if err != nil || len(rows) == 0 {
+ if err != nil {
+ return nil, err
+ }
+ break
+ }
+ // Every claimed row is handed out or dropped, so a batch may pass the cap
+ for _, c := range rows {
+ id := jobIDOf(c.once, c.data)
+ drop, err := q.marked(tx, id, now)
+ if err != nil {
+ return nil, err
+ }
+ if !drop {
+ out = append(out, c)
+ continue
+ }
+ if _, err := q.end(tx, c.id, id, contract.JobCancelled, "cancelled", nil, now); err != nil {
+ return nil, err
+ }
+ drops++
+ }
+ }
+ return out, tx.Commit()
+}
+
+// claimRows runs a claim statement and reads every row it returns; the
// caller holds the write mutex
-func (q *Queue) claimRows(n int) ([]claimed, error) {
- rows, err := q.claim.Query(time.Now().UTC(), n)
+func (q *Queue) claimRows(claim *sql.Stmt, n int) ([]claimed, error) {
+ rows, err := claim.Query(time.Now().UTC(), n)
if err != nil {
return nil, err
}
@@ -380,9 +441,9 @@ func (q *Queue) claimRows(n int) ([]claimed, error) {
for rows.Next() {
var c claimed
if q.priority {
- err = rows.Scan(&c.id, &c.priority, &c.data)
+ err = rows.Scan(&c.id, &c.priority, &c.data, &c.once)
} else {
- err = rows.Scan(&c.id, &c.data)
+ err = rows.Scan(&c.id, &c.data, &c.once)
}
if err != nil {
return nil, err
@@ -403,25 +464,45 @@ func (q *Queue) Dequeue() (any, bool) {
q.f.mu.Lock()
defer q.f.mu.Unlock()
- // Deferred last, so the rows close (and the statement ends) before unlocking
- rows, err := q.pop.Query()
+ now := time.Now()
+ if !q.anyMark(now) {
+ data, _, err := popRow(q.pop)
+ return data, err == nil
+ }
+
+ // Gated: a marked item is dropped (outcome cancelled) and the next one popped
+ tx, err := q.f.db.Begin()
if err != nil {
return nil, false
}
- defer rows.Close()
-
- var data []byte
- found := false
- for rows.Next() {
- if err := rows.Scan(&data); err != nil {
+ defer tx.Rollback()
+ pop := tx.Stmt(q.pop)
+ for range maxDrops {
+ data, once, err := popRow(pop)
+ if err != nil {
+ break
+ }
+ id := jobIDOf(once, data)
+ drop, err := q.marked(tx, id, now)
+ if err != nil {
+ return nil, false
+ }
+ if !drop {
+ return data, tx.Commit() == nil
+ }
+ if err := q.record(tx, id, contract.JobCancelled, "cancelled", nil, now); err != nil {
return nil, false
}
- found = true
- }
- if rows.Err() != nil || !found {
- return nil, false
}
- return data, true
+ tx.Commit() // keep the drops; an empty queue or the cap hands out nothing
+ return nil, false
+}
+
+// popRow runs a pop statement: the deleted row's data and job_id, or
+// sql.ErrNoRows on an empty queue
+func popRow(pop *sql.Stmt) (data []byte, once sql.NullString, err error) {
+ err = pop.QueryRow().Scan(&data, &once)
+ return data, once, err
}
// DequeueWithAckId moves the next item to processing and returns it with its
@@ -471,46 +552,77 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini
// Acknowledge marks a processing item as completed, deleting it unless
// WithRemoveOnComplete(false) is set, in the file's next group commit round
// Returns true only if exactly one processing row had this ack ID
-func (q *Queue) Acknowledge(ackID string) bool { return q.finish(ackID, q.ack) }
+func (q *Queue) Acknowledge(ackID string) bool {
+ return q.finish(ackID, contract.JobCompleted, "", nil)
+}
// Reject marks a processing item as failed for good: it is not requeued.
// The row is deleted, or with WithRemoveOnComplete(false) kept with status
-// 'failed'. Same group commit and result as Acknowledge.
-// ponytail: reason is not stored (no column); add one when something reads it.
-func (q *Queue) Reject(ackID string, reason string) bool { return q.finish(ackID, q.reject) }
+// 'failed'. reason is the error of the job's outcome. Same group commit and
+// result as Acknowledge.
+func (q *Queue) Reject(ackID string, reason string) bool {
+ return q.finish(ackID, contract.JobFailed, reason, nil)
+}
-// finish runs stmt (ack or reject) for the row of ackID.
-func (q *Queue) finish(ackID string, stmt *sql.Stmt) bool {
+// finish ends the processing row of ackID with state in the file's next
+// group commit round. A round of one runs in its own transaction: the row
+// and its outcome record commit together.
+func (q *Queue) finish(ackID string, state contract.JobState, reason string, result []byte) bool {
id, ok := parseAckID(ackID)
if !ok {
return false
}
- now := time.Now().UTC()
- return q.f.b.do(func(tx *sql.Tx) (bool, error) {
- var (
- result sql.Result
- err error
- )
- s := stmt
- if tx != nil {
- s = tx.Stmt(s)
+ now := time.Now()
+ return q.f.b.do(func(tx *sql.Tx) (ok bool, err error) {
+ if tx == nil {
+ if tx, err = q.f.db.Begin(); err != nil {
+ return false, err
+ }
+ defer func() {
+ if err == nil {
+ err = tx.Commit()
+ } else {
+ tx.Rollback()
+ }
+ ok = ok && err == nil
+ }()
}
- if q.removeOnComplete {
- result, err = s.Exec(id)
- } else {
- result, err = s.Exec(now, id)
- }
- if err != nil {
- return false, err
- }
-
// No row is an outcome, not an error: it must not fail the round
- n, err := result.RowsAffected()
- return err == nil && n == 1, err
+ return q.end(tx, id, "", state, reason, result, now)
})
}
+// end finishes processing row id with state, in tx: the row is deleted or
+// kept with state as status, and a job with a Job ID gets its outcome
+// record. jobID is the row's, when the caller knows it. Reports whether
+// the row was processing.
+func (q *Queue) end(tx *sql.Tx, id int64, jobID string, state contract.JobState, reason string, result []byte, now time.Time) (bool, error) {
+ var row *sql.Row
+ if q.removeOnComplete {
+ row = tx.Stmt(q.fin).QueryRow(id)
+ } else {
+ row = tx.Stmt(q.fin).QueryRow(string(state), now.UTC(), id)
+ }
+ var (
+ once sql.NullString
+ data []byte
+ )
+ switch err := row.Scan(&once, &data); {
+ case errors.Is(err, sql.ErrNoRows):
+ return false, nil
+ case err != nil:
+ return false, err
+ }
+ if jobID == "" {
+ jobID = jobIDOf(once, data)
+ }
+ if jobID == "" {
+ return true, nil
+ }
+ return true, q.record(tx, jobID, state, reason, result, now)
+}
+
// Len returns the number of pending items in the queue
func (q *Queue) Len() int {
var count int
@@ -543,17 +655,30 @@ func (q *Queue) Values() []any {
return items
}
-// Purge removes all items from the queue
+// Purge removes all items from the queue, with its outcome records and
+// cancel marks (spec 6.6, 6.14)
func (q *Queue) Purge() {
q.f.mu.Lock()
defer q.f.mu.Unlock()
- q.f.db.Exec(fmt.Sprintf("DELETE FROM %s", q.table))
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return
+ }
+ defer tx.Rollback()
+ for _, t := range []string{q.table, q.jobs, q.cancelled} {
+ if _, err := tx.Exec("DELETE FROM " + t); err != nil {
+ return
+ }
+ }
+ tx.Commit()
}
-// Close closes the queue and its database connection
+// Close closes the queue and drops its outcome watches and OnCancel hooks
+// without calling them. The database file stays open for the Queues.
func (q *Queue) Close() error {
q.closed.Store(true)
+ q.w.close()
return nil
}
diff --git a/queue_test.go b/queue_test.go
index bf66ab0..68775ff 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -1,7 +1,9 @@
package sqliteq
import (
+ "context"
"database/sql"
+ "errors"
"fmt"
"path/filepath"
"strconv"
@@ -9,6 +11,9 @@ import (
"sync/atomic"
"testing"
"time"
+
+ "github.com/goptics/varmq"
+ "github.com/goptics/varmq/contract"
)
func TestSQLiteQueue(t *testing.T) {
@@ -844,6 +849,9 @@ func TestWritesTakeFileMutex(t *testing.T) {
{"DequeueWithAckId", func() { q.DequeueWithAckId() }},
{"DequeueBatchWithAckIds", func() { q.DequeueBatchWithAckIds(make([]any, 2), make([]string, 2)) }},
{"Acknowledge", func() { q.Acknowledge(ack1) }},
+ {"CancelJob", func() { q.CancelJob("x") }},
+ {"gated DequeueWithAckId", func() { q.DequeueWithAckId() }},
+ {"gated Dequeue", func() { q.Dequeue() }},
{"requeueNoAck", func() { q.requeueNoAck() }},
{"Purge", func() { q.Purge() }},
}
@@ -894,3 +902,441 @@ func TestWritesTakeFileMutex(t *testing.T) {
})
}
}
+
+// otherProcess opens q's table again on a pool of its own, as a second
+// process on the same database file would.
+func otherProcess(t *testing.T, q *Queue) *Queue {
+ t.Helper()
+ db, err := sql.Open(driverName(q.f.pages), dsn(q.f.key))
+ if err != nil {
+ t.Fatal(err)
+ }
+ f := &file{key: q.f.key, db: db, pages: q.f.pages, refs: 1, handles: map[string]*handle{}, recovered: map[string]bool{}}
+ f.b = &batcher{commit: f.commitRound}
+ q2, err := newQueue(f, q.name, q.priority, WithRemoveOnComplete(q.removeOnComplete))
+ if err != nil {
+ db.Close()
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ q2.Close()
+ db.Close()
+ })
+ return q2
+}
+
+// job is a varmq-style envelope with Job ID id.
+func job(id string) []byte { return []byte(`{"id":"` + id + `","data":1}`) }
+
+// watch starts a watch of id on q; the channel gets its one outcome.
+func watch(q *Queue, id string) <-chan contract.JobOutcome {
+ ch := make(chan contract.JobOutcome, 2)
+ q.WatchOutcome(id, func(o contract.JobOutcome) { ch <- o })
+ return ch
+}
+
+func waitOutcome(t *testing.T, ch <-chan contract.JobOutcome) contract.JobOutcome {
+ t.Helper()
+ select {
+ case o := <-ch:
+ return o
+ case <-time.After(5 * time.Second):
+ t.Fatal("no outcome")
+ return contract.JobOutcome{}
+ }
+}
+
+func noOutcome(t *testing.T, ch <-chan contract.JobOutcome) {
+ t.Helper()
+ select {
+ case o := <-ch:
+ t.Fatalf("unexpected outcome %+v", o)
+ case <-time.After(4 * pollEvery):
+ }
+}
+
+// pop hands out the next item and its ack id, failing on none.
+func pop(t *testing.T, q *Queue) (string, string) {
+ t.Helper()
+ v, ok, ack := q.DequeueWithAckId()
+ if !ok {
+ t.Fatal("DequeueWithAckId found nothing")
+ }
+ return string(v.([]byte)), ack
+}
+
+// A watch reports each way a job ends: Acknowledge, Reject with its reason,
+// AcknowledgeResult with its JSON; a job without a Job ID writes nothing.
+func TestWatchOutcome(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+
+ cases := []struct {
+ id string
+ finish func(ack string) bool
+ want contract.JobOutcome
+ }{
+ {"ack", q.Acknowledge, contract.JobOutcome{ID: "ack", State: contract.JobCompleted}},
+ {"rej", func(a string) bool { return q.Reject(a, "boom") }, contract.JobOutcome{ID: "rej", State: contract.JobFailed, Error: "boom"}},
+ {"res", func(a string) bool { return q.AcknowledgeResult(a, []byte(`{"n":42}`)) },
+ contract.JobOutcome{ID: "res", State: contract.JobCompleted, Result: []byte(`{"n":42}`)}},
+ }
+ for _, c := range cases {
+ ch := watch(q, c.id)
+ q.Enqueue(job(c.id))
+ _, ack := pop(t, q)
+ noOutcome(t, ch)
+ if !c.finish(ack) {
+ t.Fatalf("%s: finish failed", c.id)
+ }
+ got := waitOutcome(t, ch)
+ if got.ID != c.want.ID || got.State != c.want.State || got.Error != c.want.Error || string(got.Result) != string(c.want.Result) {
+ t.Errorf("%s: got %+v, want %+v", c.id, got, c.want)
+ }
+ noOutcome(t, ch) // once
+ }
+
+ q.Enqueue([]byte("plain"))
+ _, ack := pop(t, q)
+ q.Acknowledge(ack)
+ if n := countJobs(t, q); n != 3 {
+ t.Errorf("Expected 3 outcome records, got %d", n)
+ }
+}
+
+func countJobs(t *testing.T, q *Queue) int {
+ t.Helper()
+ var n int
+ if err := q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.jobs).Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ return n
+}
+
+// AcknowledgeResult with invalid JSON fails and changes nothing.
+func TestAcknowledgeResultInvalid(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("a"))
+ _, ack := pop(t, q)
+ if q.AcknowledgeResult(ack, []byte(`{nope`)) {
+ t.Fatal("Expected invalid JSON to fail")
+ }
+ if n := countRows(t, q.f.db, "status = 'processing'"); n != 1 {
+ t.Errorf("Expected the row still processing, got %d", n)
+ }
+ if !q.Acknowledge(ack) {
+ t.Error("Expected Acknowledge to finish it")
+ }
+}
+
+// A job that ended before the watch is reported from its record; cancel
+// stops a watch; a closed queue watches nothing.
+func TestWatchEndedBeforeAndCancel(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("early"))
+ _, ack := pop(t, q)
+ q.Reject(ack, "late watcher")
+ if got := waitOutcome(t, watch(q, "early")); got.State != contract.JobFailed || got.Error != "late watcher" {
+ t.Errorf("got %+v", got)
+ }
+
+ ch := make(chan contract.JobOutcome, 1)
+ stop := q.WatchOutcome("never", func(o contract.JobOutcome) { ch <- o })
+ stop()
+ stop()
+ q.Enqueue(job("never"))
+ _, ack = pop(t, q)
+ q.Acknowledge(ack)
+ noOutcome(t, ch)
+
+ q.Close()
+ q.WatchOutcome("early", func(o contract.JobOutcome) { ch <- o })()
+ noOutcome(t, ch)
+}
+
+// Two queue objects on one file, as two processes: each learns of jobs the
+// other finishes, and a cancel reaches the other's pops and OnCancel.
+func TestOutcomeAcrossProcesses(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q2 := otherProcess(t, q)
+
+ ch := watch(q, "x")
+ q.Enqueue(job("x"))
+ _, ack := pop(t, q2)
+ q2.Acknowledge(ack)
+ if got := waitOutcome(t, ch); got.ID != "x" || got.State != contract.JobCompleted {
+ t.Errorf("got %+v", got)
+ }
+
+ ids := make(chan string, 4)
+ defer q2.OnCancel(func(id string) { ids <- id })()
+ q2.Enqueue(job("run"))
+ _, ack = pop(t, q2)
+ ch = watch(q2, "run")
+ if err := q.CancelJob("run"); err != nil {
+ t.Fatal(err)
+ }
+ select {
+ case id := <-ids:
+ if id != "run" {
+ t.Errorf("OnCancel got %q", id)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("OnCancel did not fire")
+ }
+ if !q2.AcknowledgeCancelled(ack, "stopped") {
+ t.Fatal("AcknowledgeCancelled failed")
+ }
+ if got := waitOutcome(t, ch); got.State != contract.JobCancelled || got.Error != "stopped" {
+ t.Errorf("got %+v", got)
+ }
+ var marks int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks)
+ if marks != 0 {
+ t.Errorf("Expected the finish to clear the mark, %d left", marks)
+ }
+ if err := q.CancelJob("run"); !errors.Is(err, contract.ErrJobEnded) {
+ t.Errorf("CancelJob of an ended job: got %v, want ErrJobEnded", err)
+ }
+}
+
+// A marked waiting job is never handed out, by any pop: it is dropped with
+// outcome cancelled and the pop goes on with the next item.
+func TestCancelWaiting(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(false))
+ defer qs.Close()
+
+ for _, id := range []string{"a", "b", "c", "d", "e"} {
+ q.Enqueue(job(id))
+ }
+ q.EnqueueOnce([]byte("raw"), "f") // a Job ID from EnqueueOnce, not JSON
+ q.Enqueue([]byte("g"))
+ chA, chF := watch(q, "a"), watch(q, "f")
+ for _, id := range []string{"a", "c", "d", "f"} {
+ if err := q.CancelJob(id); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if v, _ := pop(t, q); v != string(job("b")) {
+ t.Errorf("DequeueWithAckId: got %s, want b", v)
+ }
+ items, acks := make([]any, 2), make([]string, 2)
+ n, _, err := q.DequeueBatchWithAckIds(items, acks)
+ if err != nil || n != 2 || string(items[0].([]byte)) != string(job("e")) || string(items[1].([]byte)) != "g" {
+ t.Errorf("DequeueBatchWithAckIds: got %d %v %v", n, err, items[:n])
+ }
+ if got := waitOutcome(t, chA); got.State != contract.JobCancelled || got.Error != "cancelled" {
+ t.Errorf("got %+v", got)
+ }
+ if got := waitOutcome(t, chF); got.State != contract.JobCancelled {
+ t.Errorf("got %+v", got)
+ }
+ if n := countRows(t, q.f.db, "status = 'cancelled'"); n != 4 {
+ t.Errorf("Expected 4 kept cancelled rows, got %d", n)
+ }
+ if _, ok, _ := q.DequeueWithAckId(); ok {
+ t.Error("Expected nothing left")
+ }
+
+ // Dequeue without ack drops too
+ q.Enqueue(job("h"))
+ q.Enqueue(job("i"))
+ q.CancelJob("h")
+ if v, ok := q.Dequeue(); !ok || string(v.([]byte)) != string(job("i")) {
+ t.Errorf("Dequeue: got %v %v, want i", v, ok)
+ }
+ if got := waitOutcome(t, watch(q, "h")); got.State != contract.JobCancelled {
+ t.Errorf("got %+v", got)
+ }
+}
+
+// EnqueueOnce that stores an id drops its old outcome and its cancel mark
+// (spec 6.13, 6.14 Re-add); a dedupe hit changes neither.
+func TestReaddClearsOutcomeAndMark(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+
+ q.EnqueueOnce(job("r"), "r")
+ _, ack := pop(t, q)
+ q.Acknowledge(ack)
+ if countJobs(t, q) != 1 {
+ t.Fatal("Expected a record")
+ }
+ q.EnqueueOnce(job("r"), "r")
+ if countJobs(t, q) != 0 {
+ t.Error("Expected the re-add to drop the old record")
+ }
+ ch := watch(q, "r")
+ noOutcome(t, ch) // the earlier run is not reported
+
+ if err := q.CancelJob("r"); err != nil {
+ t.Fatal(err)
+ }
+ q.EnqueueOnce(job("r"), "r") // dedupe hit: the row is still there
+ if _, ok, _ := q.DequeueWithAckId(); ok {
+ t.Fatal("Expected the marked job dropped")
+ }
+ if got := waitOutcome(t, ch); got.State != contract.JobCancelled {
+ t.Errorf("got %+v", got)
+ }
+
+ q.CancelJob("s") // unknown id: marked
+ q.EnqueueOnce(job("s"), "s")
+ if v, _ := pop(t, q); v != string(job("s")) {
+ t.Errorf("Expected the re-added job handed out, got %s", v)
+ }
+}
+
+// Purge drops outcome records and cancel marks with the items.
+func TestPurgeClearsOutcomesAndMarks(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("done"))
+ _, ack := pop(t, q)
+ q.Acknowledge(ack)
+ q.CancelJob("later")
+ q.Purge()
+ var marks int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks)
+ if countJobs(t, q) != 0 || marks != 0 {
+ t.Errorf("Expected Purge to clear records and marks, got %d and %d", countJobs(t, q), marks)
+ }
+ q.Enqueue(job("later"))
+ if v, _ := pop(t, q); v != string(job("later")) {
+ t.Errorf("Expected the purged mark gone, got %s", v)
+ }
+}
+
+// OnCancel reports marks made after it, not before; stop ends it; an
+// expired mark neither reaches hooks nor drops items.
+func TestOnCancel(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.CancelJob("old")
+ ids := make(chan string, 4)
+ stop := q.OnCancel(func(id string) { ids <- id })
+ q.CancelJob("new")
+ select {
+ case id := <-ids:
+ if id != "new" {
+ t.Errorf("got %q, want new", id)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("OnCancel did not fire")
+ }
+ stop()
+ q.CancelJob("after")
+ select {
+ case id := <-ids:
+ t.Errorf("got %q after stop", id)
+ case <-time.After(4 * pollEvery):
+ }
+
+ q.f.db.Exec("UPDATE "+q.cancelled+" SET expires_at = ? WHERE job_id = 'old'", time.Now().Add(-time.Second).UnixMilli())
+ q.Enqueue(job("old"))
+ if v, _ := pop(t, q); v != string(job("old")) {
+ t.Errorf("Expected an expired mark to drop nothing, got %s", v)
+ }
+ if err := q.CancelJob(""); err == nil {
+ t.Error("Expected an empty id to fail")
+ }
+}
+
+// Retention trims each state group by age and count.
+func TestOutcomeRetention(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ now := time.Now()
+ tx, _ := q.f.db.Begin()
+ for i := range keepCompleted + 5 {
+ tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES (?, 'completed', ?)", "c"+strconv.Itoa(i), now.UnixMilli()+int64(i))
+ }
+ tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", now.Add(-keepFailedAge).UnixMilli())
+ tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'cancelled', ?)", now.UnixMilli())
+ if err := q.trim(tx, now); err != nil {
+ t.Fatal(err)
+ }
+ tx.Commit()
+ if n := countJobs(t, q); n != keepCompleted+1 {
+ t.Errorf("Expected %d records, got %d", keepCompleted+1, n)
+ }
+ var oldest string
+ q.f.db.QueryRow("SELECT job_id FROM " + q.jobs + " WHERE state = 'completed' ORDER BY finished_at LIMIT 1").Scan(&oldest)
+ if oldest != "c5" {
+ t.Errorf("Expected the 5 oldest completed trimmed, oldest left %s", oldest)
+ }
+}
+
+// varmq end to end: a result job added on one queue object runs on a
+// worker bound to another (another process), and its handle gets the
+// result; Cancel stops a running job and drops a waiting one.
+func TestVarmqRemoteResultAndCancel(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q2 := otherProcess(t, q)
+
+ started := make(chan struct{})
+ ran := make(chan int, 8)
+ w := varmq.NewResultWorker(func(j varmq.Job[int]) (int, error) {
+ ran <- j.Data()
+ if j.Data() < 0 {
+ close(started)
+ <-j.Context().Done()
+ return 0, j.Context().Err()
+ }
+ return j.Data() * 2, nil
+ })
+ defer w.Stop()
+
+ p := varmq.NewResultQueue[int, int](q)
+ waiting, ok := p.Add(7) // no worker yet
+ if !ok {
+ t.Fatal("Add failed")
+ }
+ if err := waiting.Cancel(); err != nil {
+ t.Fatal(err)
+ }
+ w.WithQueue(q2)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ j, _ := p.Add(21)
+ if err := j.WaitContext(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if r, err := j.Result(); err != nil || r != 42 {
+ t.Errorf("Result: got %d %v, want 42", r, err)
+ }
+ if err := waiting.WaitContext(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := waiting.Result(); !errors.Is(err, varmq.ErrJobCancelled) {
+ t.Errorf("waiting job: got %v, want ErrJobCancelled", err)
+ }
+
+ running, _ := p.Add(-1)
+ select {
+ case <-started:
+ case <-ctx.Done():
+ t.Fatal("job did not start")
+ }
+ if err := running.Cancel(); err != nil {
+ t.Fatal(err)
+ }
+ if err := running.WaitContext(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := running.Result(); !errors.Is(err, varmq.ErrJobCancelled) {
+ t.Errorf("running job: got %v, want ErrJobCancelled", err)
+ }
+ close(ran)
+ for v := range ran {
+ if v == 7 {
+ t.Error("The cancelled waiting job ran")
+ }
+ }
+}
diff --git a/registry.go b/registry.go
index 24aeb0e..517ef07 100644
--- a/registry.go
+++ b/registry.go
@@ -62,8 +62,11 @@ func driverName(pages int) string {
// dsn returns the connection string for a database file: WAL, relaxed
// fsync (safe under WAL) and a busy timeout so writers wait for each other.
+// Transactions begin IMMEDIATE: one that reads before it writes (CancelJob,
+// a gated pop) would otherwise fail with SQLITE_BUSY, not wait, when another
+// process wrote in between.
func dsn(path string) string {
- return path + "?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000"
+ return path + "?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000&_txlock=immediate"
}
// registry holds every open file in the process, keyed by resolved path.
@@ -121,6 +124,10 @@ func (f *file) release() error {
if registry.files[f.key] == f {
delete(registry.files, f.key)
}
+ // Stop the queues' polls before the pool closes under them
+ for _, h := range f.handles {
+ h.queue().Close()
+ }
registry.mu.Unlock()
// Close waits for running queries, so it runs outside the lock
From c51876e83fb0a80fa3897d92ed92f5dbc8205d93 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 21:30:05 +0600
Subject: [PATCH 03/22] docs: changelog for job outcome and cancel
---
CHANGELOG.md | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2b5eabe..ee7fa9c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,7 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
-- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is not stored.
+- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error.
+- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it every 50 ms, from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d.
+- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per pop); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended.
+- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks.
+
+### Changed
+
+- Requires Go 1.27 (varmq dependency). Transactions begin `IMMEDIATE`. A finish (`Acknowledge`, `Reject`) always runs in a transaction; a pop runs in one while any cancel mark is live.
## [0.3.0] - 2026-10-23
From 5ea8a0853d33d7176ce03dafbdeac12824f0efb3 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 21:32:11 +0600
Subject: [PATCH 04/22] fix: cancel poll era, report items left at drop cap
A poll from before an OnCancel reset no longer advances lastSeq past
the new hook's marks. A batch pop stopped at maxDrops reports one item
left so the caller pops again. Go 1.27 under Breaking.
---
CHANGELOG.md | 6 +++++-
cancel.go | 10 +++++++---
outcome.go | 5 +++--
queue.go | 51 +++++++++++++++++++++++++++++----------------------
queue_test.go | 24 ++++++++++++++++++++++++
5 files changed, 68 insertions(+), 28 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ee7fa9c..a32885c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,9 +14,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per pop); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended.
- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks.
+### Breaking
+
+- Requires Go 1.27 (was 1.24), for the varmq dependency.
+
### Changed
-- Requires Go 1.27 (varmq dependency). Transactions begin `IMMEDIATE`. A finish (`Acknowledge`, `Reject`) always runs in a transaction; a pop runs in one while any cancel mark is live.
+- Transactions begin `IMMEDIATE`. A finish (`Acknowledge`, `Reject`) always runs in a transaction; a pop runs in one while any cancel mark is live.
## [0.3.0] - 2026-10-23
diff --git a/cancel.go b/cancel.go
index ce8f1c9..e51d23f 100644
--- a/cancel.go
+++ b/cancel.go
@@ -107,6 +107,7 @@ func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) {
}
if len(w.hooks) == 0 {
w.lastSeq = last.Int64
+ w.era++
}
if w.hooks == nil {
w.hooks = map[*func(string)]struct{}{}
@@ -121,8 +122,9 @@ func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) {
}
}
-// pollCancels passes the live marks newer than seq to hooks.
-func (q *Queue) pollCancels(seq int64, hooks []func(string)) {
+// pollCancels passes the live marks newer than seq to hooks, a snapshot
+// taken in era.
+func (q *Queue) pollCancels(seq int64, era int, hooks []func(string)) {
rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? AND expires_at > ? ORDER BY seq LIMIT %d",
q.cancelled, maxDrops), seq, time.Now().UnixMilli())
if err != nil {
@@ -142,7 +144,9 @@ func (q *Queue) pollCancels(seq int64, hooks []func(string)) {
w := &q.w
w.mu.Lock()
- w.lastSeq = max(w.lastSeq, seq)
+ if w.era == era {
+ w.lastSeq = max(w.lastSeq, seq)
+ }
w.mu.Unlock()
for _, id := range ids {
for _, fn := range hooks {
diff --git a/outcome.go b/outcome.go
index 5aa91e4..dd64287 100644
--- a/outcome.go
+++ b/outcome.go
@@ -164,6 +164,7 @@ type watchers struct {
byID map[string]map[*outcomeWatch]struct{}
hooks map[*func(string)]struct{}
lastSeq int64 // newest cancel mark seq OnCancel has seen
+ era int // bumped when OnCancel resets lastSeq; a poll from an older era must not advance it
polling bool
closed bool
stop chan struct{} // closed by close
@@ -264,14 +265,14 @@ func (q *Queue) poll() {
for fn := range w.hooks {
hooks = append(hooks, *fn)
}
- seq := w.lastSeq
+ seq, era := w.lastSeq, w.era
w.mu.Unlock()
if len(ids) > 0 {
q.catchUp(ids)
}
if len(hooks) > 0 {
- q.pollCancels(seq, hooks)
+ q.pollCancels(seq, era, hooks)
}
}
}
diff --git a/queue.go b/queue.go
index 471326f..fded499 100644
--- a/queue.go
+++ b/queue.go
@@ -364,12 +364,12 @@ func (q *Queue) enqueue(item any, priority int, id string) bool {
// claimN moves up to n pending rows to processing in one statement and
// returns them in queue order
-func (q *Queue) claimN(n int) ([]claimed, error) {
+func (q *Queue) claimN(n int) ([]claimed, bool, error) {
q.f.mu.Lock()
- out, err := q.claimGated(n)
+ out, capped, err := q.claimGated(n)
q.f.mu.Unlock()
if err != nil {
- return nil, err
+ return nil, false, err
}
// RETURNING order is unspecified
@@ -379,33 +379,35 @@ func (q *Queue) claimN(n int) ([]claimed, error) {
}
return cmp.Compare(a.id, b.id)
})
- return out, nil
+ return out, capped, nil
}
// claimGated claims up to n rows; with a live cancel mark on the file it
// runs in one transaction that drops each marked row it claims (outcome
-// cancelled) and claims again, at most maxDrops drops. The caller holds the
-// write mutex.
-func (q *Queue) claimGated(n int) ([]claimed, error) {
+// cancelled) and claims again, at most maxDrops drops; capped reports that
+// it stopped there. The caller holds the write mutex.
+func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) {
now := time.Now()
if !q.anyMark(now) {
- return q.claimRows(q.claim, n)
+ out, err = q.claimRows(q.claim, n)
+ return out, false, err
}
tx, err := q.f.db.Begin()
if err != nil {
- return nil, err
+ return nil, false, err
}
defer tx.Rollback()
claim := tx.Stmt(q.claim)
- out := make([]claimed, 0, n)
- for drops := 0; len(out) < n && drops < maxDrops; {
+ out = make([]claimed, 0, n)
+ drops := 0
+ for len(out) < n && drops < maxDrops {
rows, err := q.claimRows(claim, n-len(out))
- if err != nil || len(rows) == 0 {
- if err != nil {
- return nil, err
- }
+ if err != nil {
+ return nil, false, err
+ }
+ if len(rows) == 0 {
break
}
// Every claimed row is handed out or dropped, so a batch may pass the cap
@@ -413,19 +415,19 @@ func (q *Queue) claimGated(n int) ([]claimed, error) {
id := jobIDOf(c.once, c.data)
drop, err := q.marked(tx, id, now)
if err != nil {
- return nil, err
+ return nil, false, err
}
if !drop {
out = append(out, c)
continue
}
if _, err := q.end(tx, c.id, id, contract.JobCancelled, "cancelled", nil, now); err != nil {
- return nil, err
+ return nil, false, err
}
drops++
}
}
- return out, tx.Commit()
+ return out, len(out) < n && drops >= maxDrops, tx.Commit()
}
// claimRows runs a claim statement and reads every row it returns; the
@@ -512,7 +514,7 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) {
return nil, false, ""
}
- c, err := q.claimN(1)
+ c, _, err := q.claimN(1)
if err != nil || len(c) == 0 {
return nil, false, ""
}
@@ -530,12 +532,17 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini
return 0, 0, nil
}
- c, err := q.claimN(want)
+ c, capped, err := q.claimN(want)
if err != nil {
return 0, 0, err
}
+ // At the drop cap items may wait still: report one, so the caller pops again
+ left := 0
+ if capped {
+ left = 1
+ }
if len(c) == 0 {
- return 0, 0, nil
+ return 0, left, nil
}
for i, row := range c {
items[i], ackIDs[i] = row.data, strconv.FormatInt(row.id, 10)
@@ -546,7 +553,7 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini
if err := q.countCapped.QueryRow().Scan(&remaining); err != nil {
return len(c), 1, nil
}
- return len(c), remaining, nil
+ return len(c), max(remaining, left), nil
}
// Acknowledge marks a processing item as completed, deleting it unless
diff --git a/queue_test.go b/queue_test.go
index 68775ff..680c3e5 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -1340,3 +1340,27 @@ func TestVarmqRemoteResultAndCancel(t *testing.T) {
}
}
}
+
+// A pop drops at most maxDrops marked items; at the cap it reports items
+// left, and the next pop goes on.
+func TestCancelDropCap(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ for i := range maxDrops {
+ id := "m" + strconv.Itoa(i)
+ q.Enqueue(job(id))
+ q.CancelJob(id)
+ }
+ q.Enqueue([]byte("plain"))
+
+ items, acks := make([]any, 1), make([]string, 1)
+ if n, left, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != 0 || left != 1 {
+ t.Fatalf("at the cap: got n=%d left=%d err=%v, want 0, 1", n, left, err)
+ }
+ if v, _ := pop(t, q); v != "plain" {
+ t.Errorf("got %s, want plain", v)
+ }
+ if n := countJobs(t, q); n != maxDrops {
+ t.Errorf("Expected %d cancelled records, got %d", maxDrops, n)
+ }
+}
From 9374f80d550fffc8b7ee13d0206409475759bd84 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 21:35:44 +0600
Subject: [PATCH 05/22] chore: Go 1.27 in CI and README, commitRound doc
go.mod needs 1.27 now. Note that a multi-statement op opens its own
tx in a round of one.
---
.github/workflows/sqliteq.yml | 2 +-
README.md | 4 ++--
commit.go | 3 ++-
3 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/sqliteq.yml b/.github/workflows/sqliteq.yml
index b658b4d..bf9d168 100644
--- a/.github/workflows/sqliteq.yml
+++ b/.github/workflows/sqliteq.yml
@@ -24,7 +24,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
- go-version: "1.24.1"
+ go-version: "1.27.0"
cache: true
- name: Install dependencies
diff --git a/README.md b/README.md
index d911546..6058b8b 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@
-
+
@@ -49,7 +49,7 @@
go get github.com/goptics/sqliteq
```
-Requires Go 1.24+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler.
+Requires Go 1.27+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler.
## Quick Start
diff --git a/commit.go b/commit.go
index edb4f29..be6fed2 100644
--- a/commit.go
+++ b/commit.go
@@ -10,7 +10,7 @@ import (
// code (such as a driver.Valuer): a panic there would strand the round and
// leave its transaction open. Convert user values before building it.
type op struct {
- run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit
+ run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit, or a tx of its own
done chan result // buffered 2: one lead wake-up and the outcome
}
@@ -97,6 +97,7 @@ func (b *batcher) lead() {
// transaction of its own, so one bad call fails alone. A round of one op runs
// with a nil tx, as one autocommit statement: a database/sql Tx costs a
// context and a goroutine, which a round with nothing to share does not repay.
+// An op of several statements (a finish) opens its own tx then.
func (f *file) commitRound(ops []*op) {
f.mu.Lock()
defer f.mu.Unlock()
From 7cc1fd646d48fcf039194ed810539049d38534c8 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 22:06:17 +0600
Subject: [PATCH 06/22] fix: review findings on outcome and cancel
- Outcome poll reads records by seq cursor (AUTOINCREMENT), not a
500-id sample: a burst of finishes no longer loses watches.
- Count trim spares records under 1 min old; first record of a queue
object trims; poll sweeps retention + expired marks each minute.
Index on marks expires_at.
- OnCancel: per-hook from seq, hooks picked after the read: a hook
added mid-poll gets its marks, stopped or closed ones get none. A
failed first read gives a no-op hook, not a 24 h replay.
- deliver reaches only watches started before the read (re-add).
- Drop cap per tx: claims never pass it; single pops go on past it.
- Queue names must not contain __sqliteq_.
---
CHANGELOG.md | 5 +-
cancel.go | 68 +++++++++++--------
outcome.go | 185 +++++++++++++++++++++++++++++++++++++++------------
queue.go | 54 ++++++++++-----
registry.go | 4 ++
5 files changed, 226 insertions(+), 90 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a32885c..8961d18 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,13 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error.
-- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it every 50 ms, from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d.
-- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per pop); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended.
+- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it by seq every 50 ms (new records only), from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d; the count limit spares records under 1 min old. A polling queue object also sweeps retention and expired marks once a minute.
+- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per transaction; a single-item pop goes on past that, a batch pop reports an item left); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended.
- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks.
### Breaking
- Requires Go 1.27 (was 1.24), for the varmq dependency.
+- A queue name must not contain `__sqliteq_`, in both layouts: a queue's indexes, sidecar tables and trigger use it.
### Changed
diff --git a/cancel.go b/cancel.go
index e51d23f..6b4445e 100644
--- a/cancel.go
+++ b/cancel.go
@@ -92,13 +92,16 @@ func (q *Queue) AcknowledgeCancelled(ackID, reason string) bool {
// cancel request made after the call, through any queue object on this
// database file, until stop. Marks are polled every pollEvery; each call
// runs on its own goroutine. A mark gone by the poll (the job finished or
-// was re-added) is never reported. A closed queue reports nothing.
+// was re-added) is never reported. A closed queue, or a failed first read,
+// reports nothing.
func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) {
// Read before the lock: a mark made after this read has a higher seq
var last sql.NullInt64
- _ = q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.cancelled)).Scan(&last)
+ if err := q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.cancelled)).Scan(&last); err != nil {
+ return func() {} // 0 would replay every live mark
+ }
- p := &fn
+ h := &cancelHook{fn: fn, from: last.Int64}
w := &q.w
w.mu.Lock()
defer w.mu.Unlock()
@@ -106,51 +109,63 @@ func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) {
return func() {}
}
if len(w.hooks) == 0 {
- w.lastSeq = last.Int64
- w.era++
+ w.lastSeq = h.from // no poll read marks for anyone since
}
if w.hooks == nil {
- w.hooks = map[*func(string)]struct{}{}
+ w.hooks = map[*cancelHook]struct{}{}
}
- w.hooks[p] = struct{}{}
+ w.hooks[h] = struct{}{}
w.startLocked(q)
return func() {
w.mu.Lock()
defer w.mu.Unlock()
- delete(w.hooks, p)
+ delete(w.hooks, h)
}
}
-// pollCancels passes the live marks newer than seq to hooks, a snapshot
-// taken in era.
-func (q *Queue) pollCancels(seq int64, era int, hooks []func(string)) {
+// cancelHook is one OnCancel callback; it gets marks with seq above from.
+type cancelHook struct {
+ fn func(string)
+ from int64
+}
+
+// pollCancels reads the live marks newer than seq and passes each to the
+// hooks live after the read whose from is below it, so a hook added during
+// the read still gets its marks, and a stopped one gets none.
+func (q *Queue) pollCancels(seq int64) {
rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? AND expires_at > ? ORDER BY seq LIMIT %d",
q.cancelled, maxDrops), seq, time.Now().UnixMilli())
if err != nil {
return
}
- var ids []string
+ type mark struct {
+ seq int64
+ id string
+ }
+ var marks []mark
for rows.Next() {
- var id string
- if rows.Scan(&seq, &id) != nil {
- continue
+ var m mark
+ if rows.Scan(&m.seq, &m.id) == nil {
+ marks = append(marks, m)
}
- ids = append(ids, id)
}
- if rows.Close() != nil || rows.Err() != nil {
+ if rows.Close() != nil || rows.Err() != nil || len(marks) == 0 {
return // read again from the old seq
}
w := &q.w
w.mu.Lock()
- if w.era == era {
- w.lastSeq = max(w.lastSeq, seq)
+ defer w.mu.Unlock()
+ if w.closed {
+ return
}
- w.mu.Unlock()
- for _, id := range ids {
- for _, fn := range hooks {
- go fn(id)
+ w.lastSeq = max(w.lastSeq, marks[len(marks)-1].seq)
+ for h := range w.hooks {
+ for _, m := range marks {
+ if m.seq > h.from {
+ go h.fn(m.id)
+ }
}
}
}
@@ -165,12 +180,13 @@ func (q *Queue) anyMark(now time.Time) bool {
return n == 1
}
-// marked reports whether job id has a live mark, in tx.
-func (q *Queue) marked(tx *sql.Tx, id string, now time.Time) (bool, error) {
+// marked reports whether job id has a live mark; isMarked is q.isMarked
+// in the caller's tx.
+func marked(isMarked *sql.Stmt, id string, now time.Time) (bool, error) {
if id == "" {
return false, nil
}
var n int
- err := tx.Stmt(q.isMarked).QueryRow(id, now.UnixMilli()).Scan(&n)
+ err := isMarked.QueryRow(id, now.UnixMilli()).Scan(&n)
return n == 1, err
}
diff --git a/outcome.go b/outcome.go
index dd64287..643f2a3 100644
--- a/outcome.go
+++ b/outcome.go
@@ -14,9 +14,11 @@ import (
// Job outcome (varmq spec/redis-storage.md 1.6.0, 6.13), on SQLite. Each
// finish of a job with a Job ID writes one row to __sqliteq_jobs in
-// the finish's transaction. SQLite has no pub/sub, so a watcher is
-// answered by polling that table: one goroutine per queue object, alive
-// only while it has watches or OnCancel hooks.
+// the finish's transaction, under a new AUTOINCREMENT seq. SQLite has no
+// pub/sub, so a watcher is answered by polling that table by seq (records
+// newer than the last read): one goroutine per queue object, alive only
+// while it has watches or OnCancel hooks. It also sweeps retention and
+// expired marks.
var (
_ contract.OutcomeWatcher = (*Queue)(nil)
@@ -29,9 +31,12 @@ const (
// pollEvery is how often a queue object with watches or hooks reads
// the outcome and cancel tables.
pollEvery = 50 * time.Millisecond
- // maxCatchUp caps the watched ids one poll reads; map order rotates a
- // larger set.
- maxCatchUp = 500
+ // pollBatch caps the rows one cursor read returns; a poll reads again
+ // until it gets fewer.
+ pollBatch = 1000
+ // sweepEvery is how often a polling queue object trims retention and
+ // drops expired marks.
+ sweepEvery = time.Minute
// Retention (spec 5 defaults): cancelled records count as failed.
keepCompleted = 1000
@@ -40,6 +45,9 @@ const (
keepFailedAge = 7 * 24 * time.Hour
// trimEvery is how many records a queue object writes between trims.
trimEvery = 100
+ // trimFloor: the count limit never drops a record younger than this,
+ // so a burst can't trim one before a poll reads it.
+ trimFloor = time.Minute
jobsSuffix = "__sqliteq_jobs"
cancelledSuffix = "__sqliteq_cancelled"
@@ -52,28 +60,31 @@ const (
// holds the write mutex.
func (q *Queue) initSidecars() error {
for _, s := range []string{
+ // AUTOINCREMENT: seq never reuses a deleted row's, so a seq > last
+ // cursor sees every new record and mark; REPLACE gives a new seq
`CREATE TABLE IF NOT EXISTS %[2]s (
- job_id TEXT PRIMARY KEY,
+ seq INTEGER PRIMARY KEY AUTOINCREMENT,
+ job_id TEXT NOT NULL UNIQUE,
state TEXT NOT NULL,
error TEXT,
result BLOB,
finished_at INTEGER NOT NULL
- ) WITHOUT ROWID`,
+ )`,
`CREATE INDEX IF NOT EXISTS %[4]s ON %[2]s (finished_at)`,
- // AUTOINCREMENT: seq never reuses a deleted mark's, so OnCancel's
- // seq > last cursor sees every new mark
`CREATE TABLE IF NOT EXISTS %[3]s (
seq INTEGER PRIMARY KEY AUTOINCREMENT,
job_id TEXT NOT NULL UNIQUE,
expires_at INTEGER NOT NULL
)`,
+ `CREATE INDEX IF NOT EXISTS %[6]s ON %[3]s (expires_at)`,
`CREATE TRIGGER IF NOT EXISTS %[5]s AFTER INSERT ON %[1]s WHEN NEW.job_id IS NOT NULL BEGIN
DELETE FROM %[2]s WHERE job_id = NEW.job_id;
DELETE FROM %[3]s WHERE job_id = NEW.job_id;
END`,
} {
query := fmt.Sprintf(s, q.table, q.jobs, q.cancelled,
- quoteIdent(q.name+jobsSuffix+"_finished"), quoteIdent(q.name+readdSuffix))
+ quoteIdent(q.name+jobsSuffix+"_finished"), quoteIdent(q.name+readdSuffix),
+ quoteIdent(q.name+cancelledSuffix+"_expires"))
if _, err := q.f.db.Exec(query); err != nil {
return err
}
@@ -113,14 +124,16 @@ func (q *Queue) record(tx *sql.Tx, id string, state contract.JobState, reason st
if _, err := tx.Stmt(q.unmark).Exec(id); err != nil {
return err
}
- if q.records.Add(1)%trimEvery == 0 {
+ // The first record of a queue object trims too: a short-lived process
+ // may write fewer than trimEvery
+ if q.records.Add(1)%trimEvery == 1 {
return q.trim(tx, now)
}
return nil
}
// trim applies retention: per state group, records past the age limit and
-// those past the newest N.
+// those past the newest N that are older than trimFloor.
func (q *Queue) trim(tx *sql.Tx, now time.Time) error {
for _, g := range []struct {
where string
@@ -134,8 +147,8 @@ func (q *Queue) trim(tx *sql.Tx, now time.Time) error {
now.Add(-g.age).UnixMilli()); err != nil {
return err
}
- if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %[1]s WHERE job_id IN (SELECT job_id FROM %[1]s WHERE %[2]s ORDER BY finished_at DESC LIMIT -1 OFFSET %[3]d)",
- q.jobs, g.where, g.keep)); err != nil {
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %[1]s WHERE finished_at <= ? AND job_id IN (SELECT job_id FROM %[1]s WHERE %[2]s ORDER BY finished_at DESC LIMIT -1 OFFSET %[3]d)",
+ q.jobs, g.where, g.keep), now.Add(-trimFloor).UnixMilli()); err != nil {
return err
}
}
@@ -154,6 +167,7 @@ func (q *Queue) AcknowledgeResult(ackID string, result []byte) bool {
type outcomeWatch struct {
fn func(contract.JobOutcome)
+ gen uint64 // w.gen at its start
done atomic.Bool // called or cancelled
}
@@ -162,9 +176,10 @@ type outcomeWatch struct {
type watchers struct {
mu sync.Mutex
byID map[string]map[*outcomeWatch]struct{}
- hooks map[*func(string)]struct{}
- lastSeq int64 // newest cancel mark seq OnCancel has seen
- era int // bumped when OnCancel resets lastSeq; a poll from an older era must not advance it
+ hooks map[*cancelHook]struct{}
+ lastSeq int64 // newest cancel mark seq a poll has read
+ outSeq int64 // newest outcome record seq a poll has read
+ gen uint64 // watches started; a read reaches only watches started before it
polling bool
closed bool
stop chan struct{} // closed by close
@@ -209,6 +224,23 @@ func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel fu
w.mu.Unlock()
return func() {}
}
+ if len(w.byID) == 0 {
+ // No poll moved the cursor while nothing was watched: start it at
+ // the newest record. Older ones are this watch's catchUp below.
+ w.mu.Unlock()
+ var last sql.NullInt64
+ err := q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.jobs)).Scan(&last)
+ w.mu.Lock()
+ if w.closed {
+ w.mu.Unlock()
+ return func() {}
+ }
+ if err == nil && len(w.byID) == 0 {
+ w.outSeq = last.Int64
+ }
+ }
+ w.gen++
+ o.gen = w.gen
if w.byID == nil {
w.byID = map[string]map[*outcomeWatch]struct{}{}
}
@@ -219,7 +251,7 @@ func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel fu
w.startLocked(q)
w.mu.Unlock()
- go q.catchUp([]string{id})
+ go q.catchUp([]string{id}, o.gen)
return func() {
if o.done.CompareAndSwap(false, true) {
@@ -235,12 +267,13 @@ func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel fu
}
}
-// poll reads the watched outcomes and new cancel marks every pollEvery,
-// until Close or until nothing is watched.
+// poll reads new outcome records and cancel marks every pollEvery, and
+// sweeps every sweepEvery, until Close or until nothing is watched.
func (q *Queue) poll() {
t := time.NewTicker(pollEvery)
defer t.Stop()
w := &q.w
+ swept := time.Now()
for {
select {
case <-w.stop:
@@ -254,32 +287,85 @@ func (q *Queue) poll() {
w.mu.Unlock()
return
}
- ids := make([]string, 0, min(len(w.byID), maxCatchUp))
- for id := range w.byID {
- if len(ids) == maxCatchUp {
- break
+ watching, hooks := len(w.byID) > 0, len(w.hooks) > 0
+ outSeq, seq, gen := w.outSeq, w.lastSeq, w.gen
+ w.mu.Unlock()
+
+ if watching {
+ q.pollOutcomes(outSeq, gen)
+ }
+ if hooks {
+ q.pollCancels(seq)
+ }
+ if time.Since(swept) >= sweepEvery {
+ swept = time.Now()
+ q.sweep()
+ }
+ }
+}
+
+// pollOutcomes reads the records newer than seq, in pollBatch reads, and
+// delivers the watched ones to watches up to gen.
+func (q *Queue) pollOutcomes(seq int64, gen uint64) {
+ for {
+ rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? ORDER BY seq LIMIT %d", q.jobs, pollBatch), seq)
+ if err != nil {
+ return
+ }
+ var ids []string
+ for rows.Next() {
+ var id string
+ if rows.Scan(&seq, &id) == nil {
+ ids = append(ids, id)
}
- ids = append(ids, id)
}
- hooks := make([]func(string), 0, len(w.hooks))
- for fn := range w.hooks {
- hooks = append(hooks, *fn)
+ if rows.Close() != nil || rows.Err() != nil || len(ids) == 0 {
+ return // read again from the old seq
}
- seq, era := w.lastSeq, w.era
- w.mu.Unlock()
- if len(ids) > 0 {
- q.catchUp(ids)
+ w := &q.w
+ w.mu.Lock()
+ w.outSeq = max(w.outSeq, seq)
+ watched := ids[:0]
+ for _, id := range ids {
+ if w.byID[id] != nil {
+ watched = append(watched, id)
+ }
+ }
+ w.mu.Unlock()
+ if len(watched) > 0 {
+ q.catchUp(watched, gen)
}
- if len(hooks) > 0 {
- q.pollCancels(seq, era, hooks)
+ if len(ids) < pollBatch {
+ return
}
}
}
-// catchUp delivers the ids whose outcome record exists. A failed read is
-// retried by the next poll.
-func (q *Queue) catchUp(ids []string) {
+// sweep applies retention and drops expired cancel marks (spec 5, 6.14
+// Prune), so an idle queue ages out too.
+func (q *Queue) sweep() {
+ q.f.mu.Lock()
+ defer q.f.mu.Unlock()
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return
+ }
+ defer tx.Rollback()
+ now := time.Now()
+ if q.trim(tx, now) != nil {
+ return
+ }
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE expires_at <= ?", q.cancelled), now.UnixMilli()); err != nil {
+ return
+ }
+ tx.Commit()
+}
+
+// catchUp delivers the ids whose outcome record exists to their watches
+// up to gen, those started before the read. A failed read is retried by
+// the next poll.
+func (q *Queue) catchUp(ids []string, gen uint64) {
args := make([]any, len(ids))
for i, id := range ids {
args[i] = id
@@ -304,20 +390,31 @@ func (q *Queue) catchUp(ids []string) {
rows.Close()
for _, out := range outs {
- q.deliver(out)
+ q.deliver(out, gen)
}
}
-// deliver ends every watch of out.ID: each fn runs once, on its own
-// goroutine, so a slow fn holds up no poll.
-func (q *Queue) deliver(out contract.JobOutcome) {
+// deliver ends the watches of out.ID started up to gen: each fn runs once,
+// on its own goroutine, so a slow fn holds up no poll. A watch started
+// after the read stays: the record read may be an earlier run's, which a
+// re-add since deleted (spec 6.13 Re-add).
+func (q *Queue) deliver(out contract.JobOutcome, gen uint64) {
w := &q.w
w.mu.Lock()
set := w.byID[out.ID]
- delete(w.byID, out.ID)
+ var ended []*outcomeWatch
+ for o := range set {
+ if o.gen <= gen {
+ ended = append(ended, o)
+ delete(set, o)
+ }
+ }
+ if len(set) == 0 {
+ delete(w.byID, out.ID)
+ }
w.mu.Unlock()
- for o := range set {
+ for _, o := range ended {
go func() {
if o.done.CompareAndSwap(false, true) {
o.fn(out)
diff --git a/queue.go b/queue.go
index fded499..be1fbdd 100644
--- a/queue.go
+++ b/queue.go
@@ -399,21 +399,21 @@ func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) {
}
defer tx.Rollback()
- claim := tx.Stmt(q.claim)
+ claim, isMarked := tx.Stmt(q.claim), tx.Stmt(q.isMarked)
out = make([]claimed, 0, n)
drops := 0
for len(out) < n && drops < maxDrops {
- rows, err := q.claimRows(claim, n-len(out))
+ // No more than the drops left: a batch never passes the cap
+ rows, err := q.claimRows(claim, min(n-len(out), maxDrops-drops))
if err != nil {
return nil, false, err
}
if len(rows) == 0 {
break
}
- // Every claimed row is handed out or dropped, so a batch may pass the cap
for _, c := range rows {
id := jobIDOf(c.once, c.data)
- drop, err := q.marked(tx, id, now)
+ drop, err := marked(isMarked, id, now)
if err != nil {
return nil, false, err
}
@@ -471,33 +471,45 @@ func (q *Queue) Dequeue() (any, bool) {
data, _, err := popRow(q.pop)
return data, err == nil
}
+ for {
+ data, capped, err := q.popGated(now)
+ if err != nil || data != nil || !capped {
+ return data, err == nil && data != nil
+ }
+ }
+}
- // Gated: a marked item is dropped (outcome cancelled) and the next one popped
+// popGated pops in one transaction, dropping each marked item it pops
+// (outcome cancelled), at most maxDrops; capped reports it stopped there
+// with nothing. The caller holds the write mutex.
+func (q *Queue) popGated(now time.Time) (data []byte, capped bool, err error) {
tx, err := q.f.db.Begin()
if err != nil {
- return nil, false
+ return nil, false, err
}
defer tx.Rollback()
- pop := tx.Stmt(q.pop)
+ pop, isMarked := tx.Stmt(q.pop), tx.Stmt(q.isMarked)
for range maxDrops {
data, once, err := popRow(pop)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, false, tx.Commit() // keep the drops
+ }
if err != nil {
- break
+ return nil, false, err
}
id := jobIDOf(once, data)
- drop, err := q.marked(tx, id, now)
+ drop, err := marked(isMarked, id, now)
if err != nil {
- return nil, false
+ return nil, false, err
}
if !drop {
- return data, tx.Commit() == nil
+ return data, false, tx.Commit()
}
if err := q.record(tx, id, contract.JobCancelled, "cancelled", nil, now); err != nil {
- return nil, false
+ return nil, false, err
}
}
- tx.Commit() // keep the drops; an empty queue or the cap hands out nothing
- return nil, false
+ return nil, true, tx.Commit()
}
// popRow runs a pop statement: the deleted row's data and job_id, or
@@ -514,11 +526,17 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) {
return nil, false, ""
}
- c, _, err := q.claimN(1)
- if err != nil || len(c) == 0 {
- return nil, false, ""
+ // A pop stopped at the drop cap has nothing yet: pop again, it can't
+ // report items left
+ for {
+ c, capped, err := q.claimN(1)
+ if err != nil || len(c) == 0 && !capped {
+ return nil, false, ""
+ }
+ if len(c) > 0 {
+ return c[0].data, true, strconv.FormatInt(c[0].id, 10)
+ }
}
- return c[0].data, true, strconv.FormatInt(c[0].id, 10)
}
// DequeueBatchWithAckIds implements varmq.IBatchAcknowledgeable: it moves up
diff --git a/registry.go b/registry.go
index 517ef07..5db083a 100644
--- a/registry.go
+++ b/registry.go
@@ -150,6 +150,10 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error)
}
key := foldName(name)
+ // A queue's indexes, sidecar tables and trigger are named __sqliteq_*
+ if strings.Contains(key, "__sqliteq_") {
+ return nil, fmt.Errorf("sqliteq: queue name %q must not contain __sqliteq_", name)
+ }
if h, ok := f.handles[key]; ok {
if (h.pq != nil) != priority {
return nil, fmt.Errorf("sqliteq: %q is already open as a %s", name, h.kind())
From e009366a192c7ec8c60ca7a2db6cf31c959c1072 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 22:06:17 +0600
Subject: [PATCH 07/22] test: outcome and cancel review gaps
Batched finishes (shared and fallback round), Close with live watch
and hook, hooks during a poll and stale marks, repeat cancel, gated
Dequeue to empty, kept-row ends, deliver generation, 3000-watch burst,
reserved names, sidecars on a v0.2 file, retention floor and first
record trim. Closed-queue watch check no longer vacuous.
---
commit_test.go | 50 +++++++++
migration_test.go | 6 ++
queue_test.go | 258 +++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 310 insertions(+), 4 deletions(-)
diff --git a/commit_test.go b/commit_test.go
index 6b1a7e1..54fa30d 100644
--- a/commit_test.go
+++ b/commit_test.go
@@ -491,3 +491,53 @@ func TestGroupCommitAfterClose(t *testing.T) {
}
waitAll(t, &wg, 10*time.Second)
}
+
+// Finishes that share a round write their outcomes in its transaction,
+// and again each in its own when a bad call fails the round.
+func TestGroupFinishOutcomes(t *testing.T) {
+ for _, fail := range []bool{false, true} {
+ t.Run(map[bool]string{false: "shared", true: "fallback"}[fail], func(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ for _, id := range []string{"a", "b", "c", "d"} {
+ q.Enqueue(job(id))
+ }
+ items, acks := make([]any, 4), make([]string, 4)
+ if n, _, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != 4 {
+ t.Fatalf("got %d %v", n, err)
+ }
+ rest := []func() bool{
+ func() bool { return q.Reject(acks[1], "r") },
+ func() bool { return q.AcknowledgeResult(acks[2], []byte(`1`)) },
+ func() bool { return q.AcknowledgeCancelled(acks[3], "c") },
+ }
+ if fail {
+ rest = append(rest, func() bool { return q.Enqueue(nil) })
+ }
+ got := oneRound(t, q.f, func() bool { return q.Acknowledge(acks[0]) }, rest)
+ for i, ok := range got[:4] {
+ if !ok {
+ t.Errorf("finish %d failed", i)
+ }
+ }
+ want := map[string]string{"a": "completed||", "b": "failed|r|", "c": "completed||1", "d": "cancelled|c|"}
+ rows, err := q.f.db.Query("SELECT job_id, state, COALESCE(error, ''), COALESCE(CAST(result AS TEXT), '') FROM " + q.jobs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer rows.Close()
+ n := 0
+ for rows.Next() {
+ var id, state, e, r string
+ rows.Scan(&id, &state, &e, &r)
+ if g := state + "|" + e + "|" + r; g != want[id] {
+ t.Errorf("%s: got %s, want %s", id, g, want[id])
+ }
+ n++
+ }
+ if n != 4 {
+ t.Errorf("Expected 4 records, got %d", n)
+ }
+ })
+ }
+}
diff --git a/migration_test.go b/migration_test.go
index bc94204..be706f8 100644
--- a/migration_test.go
+++ b/migration_test.go
@@ -129,6 +129,12 @@ func TestMigrateV02File(t *testing.T) {
if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) {
t.Errorf("Round %d: indexes = %v, want %v", round, got, want)
}
+ for _, side := range []string{name + jobsSuffix, name + cancelledSuffix, name + readdSuffix} {
+ var n int
+ if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = ?", side).Scan(&n); err != nil || n != 1 {
+ t.Errorf("Round %d: expected %s on the old file, got %d %v", round, side, n, err)
+ }
+ }
if round == 1 {
// The two processing rows are pending again after recovery
diff --git a/queue_test.go b/queue_test.go
index 680c3e5..c12eb7d 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"path/filepath"
+ "slices"
"strconv"
"sync"
"sync/atomic"
@@ -1052,8 +1053,11 @@ func TestWatchEndedBeforeAndCancel(t *testing.T) {
noOutcome(t, ch)
q.Close()
- q.WatchOutcome("early", func(o contract.JobOutcome) { ch <- o })()
+ q.WatchOutcome("early", func(o contract.JobOutcome) { ch <- o })
noOutcome(t, ch)
+ if err := q.CancelJob("x"); !errors.Is(err, errClosed) {
+ t.Errorf("CancelJob on a closed queue: got %v, want errClosed", err)
+ }
}
// Two queue objects on one file, as two processes: each learns of jobs the
@@ -1246,15 +1250,34 @@ func TestOnCancel(t *testing.T) {
}
}
-// Retention trims each state group by age and count.
+// Retention trims each state group by age and count; the count limit
+// spares records younger than trimFloor; a queue object's first record
+// trims.
func TestOutcomeRetention(t *testing.T) {
q, qs := openQueue(t)
defer qs.Close()
now := time.Now()
+ insert := func(tx *sql.Tx, prefix string, n int, at time.Time) {
+ for i := range n {
+ if _, err := tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES (?, 'completed', ?)", prefix+strconv.Itoa(i), at.UnixMilli()+int64(i)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+
tx, _ := q.f.db.Begin()
- for i := range keepCompleted + 5 {
- tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES (?, 'completed', ?)", "c"+strconv.Itoa(i), now.UnixMilli()+int64(i))
+ insert(tx, "fresh", keepCompleted+5, now)
+ if err := q.trim(tx, now); err != nil {
+ t.Fatal(err)
}
+ tx.Commit()
+ if n := countJobs(t, q); n != keepCompleted+5 {
+ t.Fatalf("Expected records younger than trimFloor kept, got %d", n)
+ }
+ q.f.db.Exec("DELETE FROM " + q.jobs)
+
+ tx, _ = q.f.db.Begin()
+ insert(tx, "c", keepCompleted+5, now.Add(-2*trimFloor))
tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", now.Add(-keepFailedAge).UnixMilli())
tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'cancelled', ?)", now.UnixMilli())
if err := q.trim(tx, now); err != nil {
@@ -1269,6 +1292,17 @@ func TestOutcomeRetention(t *testing.T) {
if oldest != "c5" {
t.Errorf("Expected the 5 oldest completed trimmed, oldest left %s", oldest)
}
+
+ q2 := otherProcess(t, q) // records counter at 0
+ q2.f.db.Exec("UPDATE "+q.jobs+" SET finished_at = ? WHERE job_id = 'c5'", now.Add(-keepCompletedAge).UnixMilli())
+ q2.Enqueue(job("one"))
+ _, ack := pop(t, q2)
+ q2.Acknowledge(ack)
+ var n int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.jobs + " WHERE job_id = 'c5'").Scan(&n)
+ if n != 0 {
+ t.Error("Expected a queue object's first record to trim")
+ }
}
// varmq end to end: a result job added on one queue object runs on a
@@ -1364,3 +1398,219 @@ func TestCancelDropCap(t *testing.T) {
t.Errorf("Expected %d cancelled records, got %d", maxDrops, n)
}
}
+
+// Queues.Close stops polls and drops live watches and hooks without
+// calling them.
+func TestCloseDropsWatchesAndHooks(t *testing.T) {
+ q, qs := openQueue(t)
+ q2 := otherProcess(t, q)
+ ch := watch(q, "w")
+ ids := make(chan string, 1)
+ q.OnCancel(func(id string) { ids <- id })
+ if err := qs.Close(); err != nil {
+ t.Fatal(err)
+ }
+ select {
+ case <-q.w.stop:
+ default:
+ t.Fatal("Expected Close to stop the poll")
+ }
+ q2.Enqueue(job("w"))
+ _, ack := pop(t, q2)
+ q2.Acknowledge(ack)
+ q2.CancelJob("c")
+ noOutcome(t, ch)
+ select {
+ case id := <-ids:
+ t.Errorf("hook called after Close with %q", id)
+ default:
+ }
+ if stop := q.OnCancel(func(string) { t.Error("hook on a closed queue") }); stop == nil {
+ t.Error("Expected a stop func")
+ }
+}
+
+// A hook added while a poll reads gets the marks made after its call; a
+// stopped hook gets none; a mark whose job finished (stale) reaches nobody.
+func TestOnCancelHooksDuringPoll(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ a, b := make(chan string, 4), make(chan string, 4)
+ stopA := q.OnCancel(func(id string) { a <- id })
+ q.w.mu.Lock()
+ seq := q.w.lastSeq // a poll's snapshot, taken before B
+ q.w.mu.Unlock()
+ q.CancelJob("before-b")
+ stopB := q.OnCancel(func(id string) { b <- id })
+ q.CancelJob("x")
+ q.pollCancels(seq)
+
+ // The queue's own poll may report a mark too: compare distinct ids
+ got := func(ch chan string) []string {
+ var ids []string
+ for {
+ select {
+ case id := <-ch:
+ ids = append(ids, id)
+ case <-time.After(4 * pollEvery):
+ slices.Sort(ids)
+ return slices.Compact(ids)
+ }
+ }
+ }
+ if ids := got(a); !slices.Equal(ids, []string{"before-b", "x"}) {
+ t.Errorf("hook A: got %v", ids)
+ }
+ if ids := got(b); !slices.Equal(ids, []string{"x"}) {
+ t.Errorf("hook B: got %v, want only the mark after its call", ids)
+ }
+
+ var xSeq int64
+ q.f.db.QueryRow("SELECT seq FROM " + q.cancelled + " WHERE job_id = 'x'").Scan(&xSeq)
+ stopA()
+ q.Enqueue(job("y"))
+ _, ack := pop(t, q)
+ q.CancelJob("y")
+ q.Acknowledge(ack) // the finish drops the mark: stale
+ q.CancelJob("z")
+ q.pollCancels(xSeq)
+ if ids := got(a); len(ids) != 0 {
+ t.Errorf("stopped hook got %v", ids)
+ }
+ if ids := got(b); !slices.Equal(ids, []string{"z"}) {
+ t.Errorf("hook B: got %v, want z only (y is stale)", ids)
+ }
+ stopB()
+}
+
+// A repeat CancelJob gets a new seq and expiry, so hooks hear it again.
+func TestCancelRepeatRenotifies(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ ids := make(chan string, 4)
+ defer q.OnCancel(func(id string) { ids <- id })()
+ read := func() (seq, exp int64) {
+ q.f.db.QueryRow("SELECT seq, expires_at FROM "+q.cancelled+" WHERE job_id = 'x'").Scan(&seq, &exp)
+ return
+ }
+ for i := range 2 {
+ q.CancelJob("x")
+ select {
+ case <-ids:
+ case <-time.After(5 * time.Second):
+ t.Fatalf("cancel %d not reported", i+1)
+ }
+ if i == 0 {
+ s1, e1 := read()
+ time.Sleep(2 * time.Millisecond)
+ defer func() {
+ if s2, e2 := read(); s2 <= s1 || e2 <= e1 {
+ t.Errorf("Expected a new seq and expiry: %d,%d -> %d,%d", s1, e1, s2, e2)
+ }
+ }()
+ }
+ }
+}
+
+// A gated Dequeue that drops the last item returns empty and keeps the
+// drop; the mark is gone after it.
+func TestDequeueGatedEmpty(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("x"))
+ q.CancelJob("x")
+ ch := watch(q, "x")
+ if v, ok := q.Dequeue(); ok {
+ t.Fatalf("got %v", v)
+ }
+ if got := waitOutcome(t, ch); got.State != contract.JobCancelled {
+ t.Errorf("got %+v", got)
+ }
+ var marks int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks)
+ if q.Len() != 0 || marks != 0 {
+ t.Errorf("Expected no item and no mark, got %d and %d", q.Len(), marks)
+ }
+}
+
+// With WithRemoveOnComplete(false) each end keeps its row with its status,
+// and the record keeps the result.
+func TestKeptRowEnds(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(false))
+ defer qs.Close()
+ q.Enqueue(job("r"))
+ q.Enqueue(job("c"))
+ _, ackR := pop(t, q)
+ _, ackC := pop(t, q)
+ if !q.AcknowledgeResult(ackR, []byte(`"ok"`)) || !q.AcknowledgeCancelled(ackC, "stop") {
+ t.Fatal("finish failed")
+ }
+ if countRows(t, q.f.db, "status = 'completed'") != 1 || countRows(t, q.f.db, "status = 'cancelled'") != 1 {
+ t.Error("Expected one completed and one cancelled row")
+ }
+ if got := waitOutcome(t, watch(q, "r")); string(got.Result) != `"ok"` {
+ t.Errorf("got %+v", got)
+ }
+ if got := waitOutcome(t, watch(q, "c")); got.State != contract.JobCancelled || got.Error != "stop" {
+ t.Errorf("got %+v", got)
+ }
+}
+
+// A read reaches only watches started before it: an earlier run's record,
+// read before a re-add, never ends a watch made after.
+func TestDeliverSkipsNewerWatches(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ ch := watch(q, "x")
+ q.w.mu.Lock()
+ gen := q.w.gen
+ q.w.mu.Unlock()
+ q.deliver(contract.JobOutcome{ID: "x", State: contract.JobCompleted}, gen-1)
+ noOutcome(t, ch)
+ q.deliver(contract.JobOutcome{ID: "x", State: contract.JobCompleted}, gen)
+ waitOutcome(t, ch)
+}
+
+// Every watch fires when far more jobs end between two polls than
+// retention keeps, and more ids are watched than one read used to take.
+func TestWatchBurst(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q2 := otherProcess(t, q)
+ const n = 3 * keepCompleted
+ var fired atomic.Int64
+ done := make(chan struct{})
+ for i := range n {
+ id := "b" + strconv.Itoa(i)
+ q.WatchOutcome(id, func(contract.JobOutcome) {
+ if fired.Add(1) == n {
+ close(done)
+ }
+ })
+ q.Enqueue(job(id))
+ }
+ items, acks := make([]any, 500), make([]string, 500)
+ for {
+ k, _, err := q2.DequeueBatchWithAckIds(items, acks)
+ if err != nil || k == 0 {
+ break
+ }
+ for _, a := range acks[:k] {
+ q2.Acknowledge(a)
+ }
+ }
+ select {
+ case <-done:
+ case <-time.After(20 * time.Second):
+ t.Fatalf("%d of %d watches fired", fired.Load(), n)
+ }
+}
+
+// Queue names may not hold __sqliteq_, which names a queue's own tables.
+func TestReservedQueueName(t *testing.T) {
+ qs := mustNew(t, filepath.Join(t.TempDir(), "one.db"), WithSingleFile())
+ defer qs.Close()
+ if _, err := qs.NewQueue("orders__sqliteq_jobs"); err == nil {
+ t.Error("Expected a reserved name to fail")
+ }
+}
From 665759faee32989f76ea2ba22a075cf8521483a0 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 22:21:03 +0600
Subject: [PATCH 08/22] test: sweep, single pops past the drop cap
Codecov: sweep was uncovered; DequeueWithAckId and Dequeue loop again
past maxDrops, now tested.
---
queue_test.go | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/queue_test.go b/queue_test.go
index c12eb7d..17b975d 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -1397,6 +1397,42 @@ func TestCancelDropCap(t *testing.T) {
if n := countJobs(t, q); n != maxDrops {
t.Errorf("Expected %d cancelled records, got %d", maxDrops, n)
}
+
+ // Single-item pops can't report items left: past the cap they pop again
+ for _, p := range []struct {
+ name string
+ pop func() (any, bool)
+ }{
+ {"DequeueWithAckId", func() (any, bool) { v, ok, _ := q.DequeueWithAckId(); return v, ok }},
+ {"Dequeue", q.Dequeue},
+ } {
+ for i := range maxDrops + 1 {
+ id := p.name + strconv.Itoa(i)
+ q.Enqueue(job(id))
+ q.CancelJob(id)
+ }
+ q.Enqueue([]byte("plain"))
+ if v, ok := p.pop(); !ok || string(v.([]byte)) != "plain" {
+ t.Errorf("%s past the cap: got %v %v, want plain", p.name, v, ok)
+ }
+ }
+}
+
+// sweep trims records past their age and drops expired marks.
+func TestSweep(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ old := time.Now().Add(-keepFailedAge).UnixMilli()
+ q.f.db.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", old)
+ q.f.db.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'failed', ?)", time.Now().UnixMilli())
+ q.f.db.Exec("INSERT INTO "+q.cancelled+" (job_id, expires_at) VALUES ('gone', ?)", old)
+ q.CancelJob("live")
+ q.sweep()
+ var marks int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks)
+ if countJobs(t, q) != 1 || marks != 1 {
+ t.Errorf("Expected 1 record and 1 mark left, got %d and %d", countJobs(t, q), marks)
+ }
}
// Queues.Close stops polls and drops live watches and hooks without
From f42847fb0802d1cf8ae2dab067cfe53f1b3858cc Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 22:22:20 +0600
Subject: [PATCH 09/22] build: pin contract v0.6.0, varmq main 6bc553d
varmq#179 merged; contract tagged.
---
go.mod | 4 ++--
go.sum | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/go.mod b/go.mod
index 6b04aeb..d59e374 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/goptics/sqliteq
go 1.27
require (
- github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092
- github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092
+ github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6
+ github.com/goptics/varmq/contract v0.6.0
github.com/mattn/go-sqlite3 v1.14.28
)
diff --git a/go.sum b/go.sum
index f4edda0..f800d0f 100644
--- a/go.sum
+++ b/go.sum
@@ -1,9 +1,9 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092 h1:oFPxzb1mXotIzHHKlxz+kUGO1ASrWZkg2nmkYhrgx60=
-github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E=
-github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092 h1:wWq1t3n0//1/URO+uwwdEybf6GJcVHmmtJ9lcKAHxRY=
-github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg=
+github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6 h1:0GooGJC22Z/9WxAz0f8aVzWyPkBDJhHF8JI4oYTIW+w=
+github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E=
+github.com/goptics/varmq/contract v0.6.0 h1://rpDlzSwsqvPDS98siuPDN1LyyBLFhiYeAmB7MZfbQ=
+github.com/goptics/varmq/contract v0.6.0/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg=
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
From e15946977ed7a9f7ae751cdf5fa3bdf0c983b35b Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Sun, 8 Nov 2026 22:22:20 +0600
Subject: [PATCH 10/22] test: undecodable envelope is a plain job
Bad JSON, non-string or empty id: handed out by a gated pop, finishes
as usual, no record.
---
queue_test.go | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/queue_test.go b/queue_test.go
index 17b975d..6f55614 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -1650,3 +1650,30 @@ func TestReservedQueueName(t *testing.T) {
t.Error("Expected a reserved name to fail")
}
}
+
+// An item whose envelope does not decode to a string id is a plain job:
+// with a live mark (gated pops) it is handed out, finishes as usual, and
+// writes no record.
+func TestUndecodableEnvelopeIsPlainJob(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.CancelJob("other") // keeps the pops gated
+ bad := []string{`{bad`, `{"id":5}`, `{"id":null}`, `{"id":""}`, `[1]`, "plain", `{"id":"x"`}
+ for _, b := range bad {
+ q.Enqueue([]byte(b))
+ }
+ for i, b := range bad {
+ v, ack := pop(t, q)
+ if v != b {
+ t.Fatalf("pop %d: got %q, want %q", i, v, b)
+ }
+ if i%2 == 0 {
+ q.Acknowledge(ack)
+ } else {
+ q.Reject(ack, "r")
+ }
+ }
+ if n := countJobs(t, q); n != 0 {
+ t.Errorf("Expected no records, got %d", n)
+ }
+}
From b4594f830790eb8ea1f9067c09087248626a5ddc Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:29:36 +0600
Subject: [PATCH 11/22] feat: QueueInspector on Queue and PriorityQueue
QueueID sqliteq::fifo|prio:, = first 8 hex of sha256 of
the file's resolved path. QueueStats: counts per row status in one
statement (cancelled under failed), totals from new __sqliteq_meta
(insert trigger counts adds, finish counts in its tx, Purge resets).
Jobs: pending hand-out order, processing by ack id, kept ended rows
newest first, envelope data verbatim, errors from outcome record.
---
inspect.go | 247 ++++++++++++++++++++++++++++++++++++++++++++++++++
queue.go | 30 +++++-
queue_test.go | 215 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 487 insertions(+), 5 deletions(-)
create mode 100644 inspect.go
diff --git a/inspect.go b/inspect.go
new file mode 100644
index 0000000..61b2258
--- /dev/null
+++ b/inspect.go
@@ -0,0 +1,247 @@
+package sqliteq
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "database/sql"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/goptics/varmq/contract"
+)
+
+// Queue inspection (varmq contract.QueueInspector; spec redis-storage.md 7)
+// on SQLite: counts from the queue's table, totals from __sqliteq_meta.
+// Completed and failed jobs are rows kept by WithRemoveOnComplete(false);
+// by default ended rows are deleted, so those counts and pages are empty.
+
+var (
+ _ contract.QueueInspector = (*Queue)(nil)
+ _ contract.QueueInspector = (*PriorityQueue)(nil)
+)
+
+const metaSuffix = "__sqliteq_meta"
+
+// initMeta creates the meta table (key -> integer: the totals submitted,
+// completed, failed) and the trigger that counts
+// stored adds; a repeat EnqueueOnce stores no row, so fires none. The
+// caller holds the write mutex.
+func (q *Queue) initMeta() error {
+ for _, s := range []string{
+ `CREATE TABLE IF NOT EXISTS %[2]s (key TEXT PRIMARY KEY, value INTEGER NOT NULL)`,
+ `INSERT OR IGNORE INTO %[2]s (key, value) VALUES ('submitted', 0), ('completed', 0), ('failed', 0)`,
+ `CREATE TRIGGER IF NOT EXISTS %[3]s AFTER INSERT ON %[1]s BEGIN
+ UPDATE %[2]s SET value = value + 1 WHERE key = 'submitted';
+ END`,
+ } {
+ if _, err := q.f.db.Exec(fmt.Sprintf(s, q.table, q.meta, quoteIdent(q.name+"__sqliteq_submitted"))); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// queueID is sqliteq::fifo|prio:: the first 8 hex of the
+// SHA-256 of the file's resolved path, so every process on the file agrees,
+// two files never collide, and no path shows; the name in lower case, as
+// SQLite matches tables. sqliteq has no in-memory mode: every queue is in
+// a file.
+func queueID(path, name string, priority bool) string {
+ sum := sha256.Sum256([]byte(path))
+ kind := "fifo"
+ if priority {
+ kind = "prio"
+ }
+ return "sqliteq:" + hex.EncodeToString(sum[:4]) + ":" + kind + ":" + foldName(name)
+}
+
+// QueueID implements contract.QueueInspector (see queueID).
+func (q *Queue) QueueID() string {
+ return q.id
+}
+
+// QueueStats implements contract.QueueInspector, in one statement (one
+// snapshot). Counts: rows per status, cancelled under failed. Totals:
+// adds stored and jobs finished (cancelled ones as failed; a waiting job
+// dropped for a cancel is neither) since the table was first opened by
+// this version or the last Purge. Paused is the file's, not the cache.
+// ponytail: counts scan the table; index status if kept rows grow large.
+func (q *Queue) QueueStats(ctx context.Context) (contract.QueueStats, error) {
+ if q.closed.Load() {
+ return contract.QueueStats{}, errClosed
+ }
+ var (
+ s contract.QueueStats
+ pausedAt sql.NullInt64
+ )
+ err := q.f.db.QueryRowContext(ctx, fmt.Sprintf(`SELECT
+ COUNT(*) FILTER (WHERE status = 'pending'),
+ COUNT(*) FILTER (WHERE status = 'processing'),
+ COUNT(*) FILTER (WHERE status = 'completed'),
+ COUNT(*) FILTER (WHERE status IN ('failed', 'cancelled')),
+ COALESCE((SELECT value FROM %[2]s WHERE key = 'submitted'), 0),
+ COALESCE((SELECT value FROM %[2]s WHERE key = 'completed'), 0),
+ COALESCE((SELECT value FROM %[2]s WHERE key = 'failed'), 0),
+ (SELECT value FROM %[2]s WHERE key = 'paused_at')
+ FROM %[1]s`, q.table, q.meta)).Scan(
+ &s.Counts.Pending, &s.Counts.Processing, &s.Counts.Completed, &s.Counts.Failed,
+ &s.Totals.Submitted, &s.Totals.Completed, &s.Totals.Failed, &pausedAt)
+ if err != nil {
+ return contract.QueueStats{}, err
+ }
+ s.Paused, s.PausedAt = pausedAt.Valid, pausedAt.Int64
+ return s, nil
+}
+
+// Jobs implements contract.QueueInspector: pending in hand-out order,
+// processing by ack id, completed and failed (with cancelled, state
+// cancelled) newest finish first. Not counted by sqliteq: attempts are 0
+// pending, else 1; an ended row's start is lost. errors: one entry, from
+// the job's outcome record, when it has one.
+// ponytail: ended pages sort by updated_at unindexed; fine until kept rows
+// grow large.
+func (q *Queue) Jobs(ctx context.Context, state contract.JobState, offset, limit int) ([]contract.JobRecord, error) {
+ where, order := "", "id"
+ switch state {
+ case contract.JobPending:
+ where, order = "status = 'pending'", q.order
+ case contract.JobProcessing:
+ where = "status = 'processing'"
+ case contract.JobCompleted:
+ where, order = "status = 'completed'", "updated_at DESC, id DESC"
+ case contract.JobFailed:
+ where, order = "status IN ('failed', 'cancelled')", "updated_at DESC, id DESC"
+ default:
+ return nil, fmt.Errorf("sqliteq: unknown job state %q", state)
+ }
+ if q.closed.Load() {
+ return nil, errClosed
+ }
+ jobs := []contract.JobRecord{}
+ if limit <= 0 {
+ return jobs, nil
+ }
+
+ rows, err := q.f.db.QueryContext(ctx, fmt.Sprintf("SELECT id, data, job_id, status, created_at, updated_at FROM %s WHERE %s ORDER BY %s LIMIT ? OFFSET ?",
+ q.table, where, order), limit, max(0, offset))
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ var ended []int // failed and cancelled jobs with a Job ID
+ for rows.Next() {
+ var (
+ id int64
+ data []byte
+ once sql.NullString
+ status string
+ created, updated any
+ )
+ if err := rows.Scan(&id, &data, &once, &status, &created, &updated); err != nil {
+ return nil, err
+ }
+ j := contract.JobRecord{ID: jobIDOf(once, data), State: contract.JobState(status)}
+ j.Payload, j.EnqueuedAt = unwrap(data)
+ if j.EnqueuedAt == 0 {
+ j.EnqueuedAt = unixMilli(created)
+ }
+ switch state {
+ case contract.JobPending:
+ case contract.JobProcessing:
+ j.StartedAt, j.Attempts = unixMilli(updated), 1
+ default:
+ j.FinishedAt, j.Attempts = unixMilli(updated), 1
+ if j.ID != "" && state == contract.JobFailed {
+ ended = append(ended, len(jobs))
+ }
+ }
+ // Spec 3.1: a handed-out job without an id goes by its ack id
+ if j.ID == "" && state != contract.JobPending {
+ j.ID = strconv.FormatInt(id, 10)
+ }
+ jobs = append(jobs, j)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ if len(ended) > 0 {
+ if err := q.addErrors(ctx, jobs, ended); err != nil {
+ return nil, err
+ }
+ }
+ return jobs, nil
+}
+
+// addErrors gives each jobs[i] of ended its outcome record's error, when
+// the record has its state; retention may have trimmed it.
+func (q *Queue) addErrors(ctx context.Context, jobs []contract.JobRecord, ended []int) error {
+ args := make([]any, len(ended))
+ for i, at := range ended {
+ args[i] = jobs[at].ID
+ }
+ rows, err := q.f.db.QueryContext(ctx, fmt.Sprintf("SELECT job_id, state, error FROM %s WHERE job_id IN (%s) AND error IS NOT NULL",
+ q.jobs, strings.Repeat("?, ", len(args)-1)+"?"), args...)
+ if err != nil {
+ return err
+ }
+ defer rows.Close()
+
+ type outcome struct{ state, error string }
+ byID := map[string]outcome{}
+ for rows.Next() {
+ var id string
+ var o outcome
+ if err := rows.Scan(&id, &o.state, &o.error); err != nil {
+ return err
+ }
+ byID[id] = o
+ }
+ if err := rows.Err(); err != nil {
+ return err
+ }
+ for _, at := range ended {
+ j := &jobs[at]
+ if o, ok := byID[j.ID]; ok && o.state == string(j.State) {
+ j.Errors = []contract.JobError{{Attempt: 1, At: j.FinishedAt, Error: o.error}}
+ }
+ }
+ return nil
+}
+
+// unixMilli is a TIMESTAMP column read back as Unix ms; 0 when NULL or
+// not a time.
+func unixMilli(v any) int64 {
+ if t, ok := v.(time.Time); ok {
+ return t.UnixMilli()
+ }
+ return 0
+}
+
+// unwrap returns the payload and enqueue time of item: an envelope's data
+// and enqueued_at, or for anything else the item itself if it is JSON, else
+// the item as a JSON string. The bytes are kept as stored, never re-encoded.
+func unwrap(item []byte) (json.RawMessage, int64) {
+ var env struct {
+ Status json.RawMessage `json:"status"`
+ Data json.RawMessage `json:"data"`
+ EnqueuedAt json.RawMessage `json:"enqueued_at"`
+ }
+ // An envelope has status and data (spec section 3). A field is nil only
+ // when its key is absent: a JSON null still sets it to "null".
+ if bytes.HasPrefix(bytes.TrimSpace(item), []byte("{")) && json.Unmarshal(item, &env) == nil && env.Status != nil && env.Data != nil {
+ var at int64
+ _ = json.Unmarshal(env.EnqueuedAt, &at)
+ return env.Data, at
+ }
+ if json.Valid(item) {
+ return item, 0
+ }
+ s, _ := json.Marshal(string(item))
+ return s, 0
+}
diff --git a/queue.go b/queue.go
index be1fbdd..a63f0c2 100644
--- a/queue.go
+++ b/queue.go
@@ -29,12 +29,14 @@ type Queue struct {
closed atomic.Bool
jobs, cancelled string // quoted outcome and cancel mark tables
+ meta string // quoted totals table
+ id string // QueueID
w watchers // outcome watches and OnCancel hooks
records atomic.Int64 // outcome records written, for trims
// Prepared once per handle; they close with the file's pool
insert, claim, pop, fin, count, countCapped, values *sql.Stmt
- putJob, unmark, hasMark, isMarked *sql.Stmt
+ putJob, unmark, hasMark, isMarked, tally *sql.Stmt
}
// claimed is one row a claim moved to processing
@@ -58,6 +60,8 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro
removeOnComplete: true, // Default to removing completed items
jobs: quoteIdent(name + jobsSuffix),
cancelled: quoteIdent(name + cancelledSuffix),
+ meta: quoteIdent(name + metaSuffix),
+ id: queueID(f.key, name, priority),
}
q.w.stop = make(chan struct{})
if priority {
@@ -100,6 +104,9 @@ func (q *Queue) initSchema() error {
if err := q.initSidecars(); err != nil {
return fmt.Errorf("failed to create outcome tables: %w", err)
}
+ if err := q.initMeta(); err != nil {
+ return fmt.Errorf("failed to create meta table: %w", err)
+ }
return nil
}
@@ -241,6 +248,7 @@ func (q *Queue) prepare() (err error) {
{&q.count, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", q.table)},
{&q.countCapped, fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE status = 'pending' LIMIT %d)", q.table, remainingCap)},
{&q.values, fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)},
+ {&q.tally, fmt.Sprintf("UPDATE %s SET value = value + 1 WHERE key = ?", q.meta)},
} {
if *s.stmt, err = q.f.db.Prepare(s.query); err != nil {
return err
@@ -252,7 +260,7 @@ func (q *Queue) prepare() (err error) {
// closeStmts closes the statements prepared so far
func (q *Queue) closeStmts() {
- for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.fin, q.count, q.countCapped, q.values, q.putJob, q.unmark, q.hasMark, q.isMarked} {
+ for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.fin, q.count, q.countCapped, q.values, q.putJob, q.unmark, q.hasMark, q.isMarked, q.tally} {
if s != nil {
s.Close()
}
@@ -614,7 +622,16 @@ func (q *Queue) finish(ackID string, state contract.JobState, reason string, res
}()
}
// No row is an outcome, not an error: it must not fail the round
- return q.end(tx, id, "", state, reason, result, now)
+ if ok, err = q.end(tx, id, "", state, reason, result, now); !ok || err != nil {
+ return ok, err
+ }
+ // Totals: cancelled counts as failed (spec 6.14)
+ total := "failed"
+ if state == contract.JobCompleted {
+ total = "completed"
+ }
+ _, err = tx.Stmt(q.tally).Exec(total)
+ return err == nil, err
})
}
@@ -680,8 +697,8 @@ func (q *Queue) Values() []any {
return items
}
-// Purge removes all items from the queue, with its outcome records and
-// cancel marks (spec 6.6, 6.14)
+// Purge removes all items from the queue, with its outcome records, cancel
+// marks and totals (spec 6.6, 6.14)
func (q *Queue) Purge() {
q.f.mu.Lock()
defer q.f.mu.Unlock()
@@ -696,6 +713,9 @@ func (q *Queue) Purge() {
return
}
}
+ if _, err := tx.Exec("UPDATE " + q.meta + " SET value = 0 WHERE key <> 'paused_at'"); err != nil {
+ return
+ }
tx.Commit()
}
diff --git a/queue_test.go b/queue_test.go
index 6f55614..f565c6c 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -2,12 +2,15 @@ package sqliteq
import (
"context"
+ "crypto/sha256"
"database/sql"
+ "encoding/hex"
"errors"
"fmt"
"path/filepath"
"slices"
"strconv"
+ "strings"
"sync"
"sync/atomic"
"testing"
@@ -1677,3 +1680,215 @@ func TestUndecodableEnvelopeIsPlainJob(t *testing.T) {
t.Errorf("Expected no records, got %d", n)
}
}
+
+// QueueID is sqliteq:<8 hex of the file's path>:fifo|prio:: one file agrees in every process, two files never collide.
+func TestQueueID(t *testing.T) {
+ dir := t.TempDir()
+ qs := mustNew(t, dir)
+ defer qs.Close()
+ q, _ := qs.NewQueue("Jobs")
+ pq, _ := qs.NewPriorityQueue("Prio")
+ sum := sha256.Sum256([]byte(q.f.key))
+ if want := "sqliteq:" + hex.EncodeToString(sum[:4]) + ":fifo:jobs"; q.QueueID() != want {
+ t.Errorf("got %q, want %q", q.QueueID(), want)
+ }
+ if id := pq.QueueID(); !strings.HasPrefix(id, "sqliteq:") || !strings.HasSuffix(id, ":prio:prio") {
+ t.Errorf("priority queue id %q", id)
+ }
+ if otherProcess(t, q).QueueID() != q.QueueID() {
+ t.Error("Expected one id for one file in every process")
+ }
+
+ // Same table name, other file
+ single := mustNew(t, filepath.Join(dir, "single.db"), WithSingleFile())
+ defer single.Close()
+ q2, _ := single.NewQueue("jobs")
+ if q2.QueueID() == q.QueueID() {
+ t.Error("Expected two files to have two ids")
+ }
+ if strings.Contains(q2.QueueID(), dir) {
+ t.Error("Expected no path in the id")
+ }
+}
+
+// stats reads QueueStats, failing on an error.
+func stats(t *testing.T, q *Queue) contract.QueueStats {
+ t.Helper()
+ s, err := q.QueueStats(context.Background())
+ if err != nil {
+ t.Fatal(err)
+ }
+ return s
+}
+
+// QueueStats counts rows per state (cancelled under failed), totals every
+// add and finish, and survives no Purge; the default deletes ended rows.
+func TestQueueStats(t *testing.T) {
+ for _, keep := range []bool{false, true} {
+ t.Run(fmt.Sprintf("keep=%t", keep), func(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(!keep))
+ defer qs.Close()
+ for _, id := range []string{"a", "b", "c", "d", "e"} {
+ q.Enqueue(job(id))
+ }
+ q.EnqueueOnce(job("f"), "f")
+ q.EnqueueOnce(job("f"), "f") // repeat: not submitted
+ q.CancelJob("d")
+
+ _, a := pop(t, q)
+ _, b := pop(t, q)
+ _, c := pop(t, q)
+ pop(t, q) // drops d (cancelled, in no total), hands out e
+ q.Acknowledge(a)
+ q.Reject(b, "boom")
+ q.AcknowledgeCancelled(c, "stop")
+
+ kept := int64(0)
+ if keep {
+ kept = 1
+ }
+ want := contract.QueueStats{
+ Counts: contract.JobCounts{Pending: 1, Processing: 1, Completed: kept, Failed: 3 * kept},
+ Totals: contract.JobTotals{Submitted: 6, Completed: 1, Failed: 2},
+ }
+ if got := stats(t, q); got != want {
+ t.Errorf("got %+v, want %+v", got, want)
+ }
+ if got := stats(t, otherProcess(t, q)); got != want {
+ t.Errorf("other process: got %+v, want %+v", got, want)
+ }
+
+ q.Purge()
+ if got := stats(t, q); got != (contract.QueueStats{}) {
+ t.Errorf("after Purge: got %+v", got)
+ }
+ })
+ }
+}
+
+// jobs reads one page of Jobs, failing on an error.
+func jobs(t *testing.T, q *Queue, state contract.JobState, offset, limit int) []contract.JobRecord {
+ t.Helper()
+ j, err := q.Jobs(context.Background(), state, offset, limit)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return j
+}
+
+func ids(js []contract.JobRecord) []string {
+ out := make([]string, len(js))
+ for i, j := range js {
+ out[i] = j.ID
+ }
+ return out
+}
+
+// Jobs pages each state: pending in hand-out order, processing by ack id,
+// completed and failed (with cancelled) newest first; payload is the
+// envelope's data, verbatim.
+func TestJobs(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(false))
+ defer qs.Close()
+ before := time.Now().UnixMilli()
+ q.Enqueue([]byte(`{"id":"a","status":"Created","data":{"n":1.000000000000001},"enqueued_at":7}`))
+ q.Enqueue([]byte("plain"))
+ q.Enqueue([]byte(`[1,2]`))
+ for _, id := range []string{"b", "c", "d", "e", "f"} {
+ q.Enqueue(job(id))
+ }
+ if got := ids(jobs(t, q, contract.JobPending, 0, 100)); !slices.Equal(got, []string{"a", "", "", "b", "c", "d", "e", "f"}) {
+ t.Fatalf("pending: got %q", got)
+ }
+ page := jobs(t, q, contract.JobPending, 0, 3)
+ if string(page[0].Payload) != `{"n":1.000000000000001}` || page[0].EnqueuedAt != 7 || page[0].Attempts != 0 || page[0].StartedAt != 0 {
+ t.Errorf("envelope: got %+v", page[0])
+ }
+ if string(page[1].Payload) != `"plain"` || page[1].EnqueuedAt < before {
+ t.Errorf("plain bytes: got %+v (enqueued_at from the row)", page[1])
+ }
+ if string(page[2].Payload) != `[1,2]` {
+ t.Errorf("JSON item: got %s", page[2].Payload)
+ }
+ if got := ids(jobs(t, q, contract.JobPending, 6, 5)); !slices.Equal(got, []string{"e", "f"}) {
+ t.Errorf("offset 6: got %q", got)
+ }
+
+ acks := map[string]string{}
+ for range 7 {
+ v, ack := pop(t, q)
+ acks[v] = ack
+ }
+ plainAck := acks["plain"]
+ q.Acknowledge(acks[`{"id":"a","status":"Created","data":{"n":1.000000000000001},"enqueued_at":7}`])
+ q.Reject(acks["plain"], "no id")
+ q.Reject(acks[string(job("b"))], "boom")
+ q.AcknowledgeCancelled(acks[string(job("c"))], "stopped")
+ q.Acknowledge(acks[string(job("d"))])
+
+ // A running job without an id shows its ack id
+ proc := jobs(t, q, contract.JobProcessing, 0, 10)
+ if got := ids(proc); !slices.Equal(got, []string{acks["[1,2]"], "e"}) {
+ t.Errorf("processing: got %q", got)
+ }
+ for _, j := range proc {
+ if j.State != contract.JobProcessing || j.Attempts != 1 || j.StartedAt < before || j.FinishedAt != 0 {
+ t.Errorf("processing job: got %+v", j)
+ }
+ }
+
+ if got := ids(jobs(t, q, contract.JobCompleted, 0, 10)); !slices.Equal(got, []string{"d", "a"}) {
+ t.Errorf("completed: got %q", got)
+ }
+ failed := jobs(t, q, contract.JobFailed, 0, 10)
+ if got := ids(failed); !slices.Equal(got, []string{"c", "b", plainAck}) {
+ t.Fatalf("failed: got %q", got)
+ }
+ if failed[0].State != contract.JobCancelled || len(failed[0].Errors) != 1 || failed[0].Errors[0].Error != "stopped" {
+ t.Errorf("cancelled: got %+v", failed[0])
+ }
+ if b := failed[1]; b.State != contract.JobFailed || b.FinishedAt < before || b.Attempts != 1 ||
+ !slices.Equal(b.Errors, []contract.JobError{{Attempt: 1, At: b.FinishedAt, Error: "boom"}}) {
+ t.Errorf("failed: got %+v", b)
+ }
+ if len(failed[2].Errors) != 0 {
+ t.Errorf("Expected no errors without an outcome record, got %+v", failed[2].Errors)
+ }
+ if got := ids(jobs(t, q, contract.JobFailed, 1, 1)); !slices.Equal(got, []string{"b"}) {
+ t.Errorf("failed page 2: got %q", got)
+ }
+
+ if j := jobs(t, q, contract.JobPending, 0, 0); j == nil || len(j) != 0 {
+ t.Errorf("limit 0: got %#v", j)
+ }
+ if _, err := q.Jobs(context.Background(), "bogus", 0, 10); err == nil {
+ t.Error("Expected an unknown state to fail")
+ }
+}
+
+// With the default WithRemoveOnComplete(true) ended jobs are not kept, so
+// their pages are empty.
+func TestJobsRemoveOnComplete(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("a"))
+ _, ack := pop(t, q)
+ q.Acknowledge(ack)
+ if j := jobs(t, q, contract.JobCompleted, 0, 10); len(j) != 0 {
+ t.Errorf("got %+v", j)
+ }
+}
+
+// A closed queue answers stats and jobs with an error, as varmq lists it.
+func TestInspectClosed(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Close()
+ if _, err := q.QueueStats(context.Background()); err == nil {
+ t.Error("Expected QueueStats to fail on a closed queue")
+ }
+ if _, err := q.Jobs(context.Background(), contract.JobPending, 0, 1); err == nil {
+ t.Error("Expected Jobs to fail on a closed queue")
+ }
+}
From 46c3caa5e4dc17f1d8983f92f709057556e1fc80 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:29:59 +0600
Subject: [PATCH 12/22] feat: durable queue pause (varmq QueuePauser,
PauseNotifier)
Pause stores row paused_at in __sqliteq_meta; every pop statement
selects nothing while it exists, in every process on the file, across
restarts and Purge. An empty pop reads the row: refused, Paused set,
PauseRefusals moves, batch reports 0 left. While Paused, the poll reads
the row every pollEvery to see a Resume from another process.
OnPauseChange hooks run outside package locks. A queue opened on a
paused file starts paused.
---
inspect.go | 2 +-
outcome.go | 14 ++-
pause.go | 150 +++++++++++++++++++++++++++++
priority_queue_test.go | 37 ++++++++
queue.go | 127 +++++++++++++++++--------
queue_test.go | 208 +++++++++++++++++++++++++++++++++++++++++
registry.go | 1 +
7 files changed, 493 insertions(+), 46 deletions(-)
create mode 100644 pause.go
diff --git a/inspect.go b/inspect.go
index 61b2258..b38d4aa 100644
--- a/inspect.go
+++ b/inspect.go
@@ -28,7 +28,7 @@ var (
const metaSuffix = "__sqliteq_meta"
// initMeta creates the meta table (key -> integer: the totals submitted,
-// completed, failed) and the trigger that counts
+// completed, failed; paused_at while paused) and the trigger that counts
// stored adds; a repeat EnqueueOnce stores no row, so fires none. The
// caller holds the write mutex.
func (q *Queue) initMeta() error {
diff --git a/outcome.go b/outcome.go
index 643f2a3..d5589a0 100644
--- a/outcome.go
+++ b/outcome.go
@@ -172,7 +172,8 @@ type outcomeWatch struct {
}
// watchers are a queue object's outcome watches and OnCancel hooks, and the
-// state of its one poll goroutine. mu is never held across a query.
+// state of its one poll goroutine, which also polls the pause. mu is never
+// held across a query.
type watchers struct {
mu sync.Mutex
byID map[string]map[*outcomeWatch]struct{}
@@ -267,8 +268,9 @@ func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel fu
}
}
-// poll reads new outcome records and cancel marks every pollEvery, and
-// sweeps every sweepEvery, until Close or until nothing is watched.
+// poll reads new outcome records, cancel marks and, while the cache says
+// paused, the pause row every pollEvery, and sweeps every sweepEvery,
+// until Close or until nothing is watched and the cache says running.
func (q *Queue) poll() {
t := time.NewTicker(pollEvery)
defer t.Stop()
@@ -282,7 +284,8 @@ func (q *Queue) poll() {
}
w.mu.Lock()
- if len(w.byID) == 0 && len(w.hooks) == 0 {
+ paused := q.pz.paused.Load()
+ if len(w.byID) == 0 && len(w.hooks) == 0 && !paused {
w.polling = false
w.mu.Unlock()
return
@@ -297,6 +300,9 @@ func (q *Queue) poll() {
if hooks {
q.pollCancels(seq)
}
+ if paused {
+ q.pollPause()
+ }
if time.Since(swept) >= sweepEvery {
swept = time.Now()
q.sweep()
diff --git a/pause.go b/pause.go
new file mode 100644
index 0000000..30451af
--- /dev/null
+++ b/pause.go
@@ -0,0 +1,150 @@
+package sqliteq
+
+import (
+ "database/sql"
+ "errors"
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "time"
+)
+
+// Queue pause (varmq QueuePauser and PauseNotifier; spec redis-storage.md
+// 6.7, 6.8) on SQLite. The flag is row paused_at (ms) of
+// __sqliteq_meta: every pop statement selects nothing while it
+// exists, so a Pause from any process on the file stops hand-outs at once,
+// and it outlives restarts. Paused is this object's cache: set by its own
+// Pause and Resume, by a pop the row refused, and, while it is true, by
+// the poll, which reads the row every pollEvery, so a Resume from another
+// process reaches it within pollEvery.
+
+// pauseState is a Queue's Paused cache and OnPauseChange hooks.
+type pauseState struct {
+ paused atomic.Bool
+ gen atomic.Uint64 // moves at each Pause and Resume
+ refusals atomic.Uint64 // pops refused for a pause
+ mu sync.Mutex
+ hooks map[*func(bool)]struct{}
+}
+
+// Pause stops every pop of the queue, in every process on the file, until
+// Resume. Adds, acks, rejects and recovery work as before; Purge keeps the
+// pause. Idempotent: the first pause time is kept. It returns once the row
+// is stored.
+func (q *Queue) Pause() error {
+ return q.setPauseRow(true, fmt.Sprintf("INSERT OR IGNORE INTO %s (key, value) VALUES ('paused_at', ?)", q.meta), time.Now().UnixMilli())
+}
+
+// Resume undoes Pause. Idempotent.
+func (q *Queue) Resume() error {
+ return q.setPauseRow(false, fmt.Sprintf("DELETE FROM %s WHERE key = 'paused_at'", q.meta))
+}
+
+// setPauseRow runs a Pause or Resume write and sets the cache to p.
+func (q *Queue) setPauseRow(p bool, query string, args ...any) error {
+ if q.closed.Load() {
+ return errClosed
+ }
+ q.f.mu.Lock()
+ _, err := q.f.db.Exec(query, args...)
+ q.f.mu.Unlock()
+ if err != nil {
+ return err
+ }
+ q.pz.gen.Add(1)
+ q.setPaused(p)
+ return nil
+}
+
+// Paused reports whether the queue is paused, as this object last saw it.
+// No I/O.
+func (q *Queue) Paused() bool {
+ return q.pz.paused.Load()
+}
+
+// PauseRefusals counts this object's pops refused for a pause (varmq's
+// optional QueuePauser method), so varmq tells a refusal racing a Resume
+// from a failed pop.
+func (q *Queue) PauseRefusals() uint64 {
+ return q.pz.refusals.Load()
+}
+
+// OnPauseChange implements varmq.PauseNotifier: fn gets the cache each time
+// this object sees it change (its own Pause or Resume, a refused pop, the
+// poll), until cancel. fn runs on the goroutine that saw the change, never
+// under a lock of this package, and must not block; racing changes can
+// reach fn out of order, never with a value the cache did not hold.
+func (q *Queue) OnPauseChange(fn func(paused bool)) (cancel func()) {
+ h := &fn
+ q.pz.mu.Lock()
+ if q.pz.hooks == nil {
+ q.pz.hooks = map[*func(bool)]struct{}{}
+ }
+ q.pz.hooks[h] = struct{}{}
+ q.pz.mu.Unlock()
+
+ return func() {
+ q.pz.mu.Lock()
+ delete(q.pz.hooks, h)
+ q.pz.mu.Unlock()
+ }
+}
+
+// setPaused updates the cache; on a change it starts the poll (paused) and
+// calls the hooks.
+func (q *Queue) setPaused(p bool) {
+ if q.pz.paused.Swap(p) == p {
+ return
+ }
+ if p {
+ w := &q.w
+ w.mu.Lock()
+ if !w.closed {
+ w.startLocked(q)
+ }
+ w.mu.Unlock()
+ }
+
+ q.pz.mu.Lock()
+ fns := make([]func(bool), 0, len(q.pz.hooks))
+ for h := range q.pz.hooks {
+ fns = append(fns, *h)
+ }
+ q.pz.mu.Unlock()
+ for _, fn := range fns {
+ fn(p)
+ }
+}
+
+// observe applies p, read by a call that began at gen g, unless a Pause or
+// Resume of this object came since: p may then be older than what it set.
+func (q *Queue) observe(g uint64, p bool) {
+ if q.pz.gen.Load() == g {
+ q.setPaused(p)
+ }
+}
+
+// refused records a pop refused for a pause, by a call that began at gen g.
+func (q *Queue) refused(g uint64) {
+ q.pz.refusals.Add(1)
+ q.observe(g, true)
+}
+
+// pollPause reads the pause row for the poll.
+func (q *Queue) pollPause() {
+ g := q.pz.gen.Load()
+ if paused, err := readPaused(q.pausedAt); err == nil {
+ q.observe(g, paused)
+ }
+}
+
+// readPaused reports whether the pause row exists, through stmt
+// (q.pausedAt, maybe in a tx).
+func readPaused(stmt *sql.Stmt) (bool, error) {
+ var at int64
+ err := stmt.QueryRow().Scan(&at)
+ if errors.Is(err, sql.ErrNoRows) {
+ return false, nil
+ }
+ return err == nil, err
+}
diff --git a/priority_queue_test.go b/priority_queue_test.go
index a7cadcf..327f748 100644
--- a/priority_queue_test.go
+++ b/priority_queue_test.go
@@ -3,6 +3,7 @@ package sqliteq
import (
"errors"
"fmt"
+ "slices"
"strconv"
"testing"
@@ -697,3 +698,39 @@ func TestPriorityOutcomeAndCancel(t *testing.T) {
t.Errorf("Expected the re-added job first, got %s", v)
}
}
+
+// A priority queue lists pending jobs in hand-out order (priority, then
+// insert), counts as Queue does, and pauses every pop.
+func TestPriorityInspectAndPause(t *testing.T) {
+ qs := mustNew(t, t.TempDir())
+ defer qs.Close()
+ pq, err := qs.NewPriorityQueue("pq")
+ if err != nil {
+ t.Fatal(err)
+ }
+ pq.Enqueue(job("low"), 5)
+ pq.Enqueue(job("high"), 1)
+ pq.EnqueueOnce(job("mid"), 3, "mid")
+ pq.Enqueue(job("high2"), 1)
+
+ if got := ids(jobs(t, pq.Queue, contract.JobPending, 0, 10)); !slices.Equal(got, []string{"high", "high2", "mid", "low"}) {
+ t.Errorf("pending: got %q", got)
+ }
+ if got := ids(jobs(t, pq.Queue, contract.JobPending, 1, 2)); !slices.Equal(got, []string{"high2", "mid"}) {
+ t.Errorf("page: got %q", got)
+ }
+ if s := stats(t, pq.Queue); s.Counts.Pending != 4 || s.Totals.Submitted != 4 {
+ t.Errorf("stats: got %+v", s)
+ }
+
+ if err := pq.Pause(); err != nil {
+ t.Fatal(err)
+ }
+ refusedPops(t, pq.Queue)
+ if err := pq.Resume(); err != nil {
+ t.Fatal(err)
+ }
+ if v, _ := pop(t, pq.Queue); v != string(job("high")) {
+ t.Errorf("got %s, want high", v)
+ }
+}
diff --git a/queue.go b/queue.go
index a63f0c2..6f3aa70 100644
--- a/queue.go
+++ b/queue.go
@@ -29,14 +29,15 @@ type Queue struct {
closed atomic.Bool
jobs, cancelled string // quoted outcome and cancel mark tables
- meta string // quoted totals table
+ meta string // quoted totals and pause table
id string // QueueID
- w watchers // outcome watches and OnCancel hooks
+ w watchers // outcome watches, OnCancel hooks, pause poll
+ pz pauseState // Paused cache and OnPauseChange hooks
records atomic.Int64 // outcome records written, for trims
// Prepared once per handle; they close with the file's pool
insert, claim, pop, fin, count, countCapped, values *sql.Stmt
- putJob, unmark, hasMark, isMarked, tally *sql.Stmt
+ putJob, unmark, hasMark, isMarked, pausedAt, tally *sql.Stmt
}
// claimed is one row a claim moved to processing
@@ -83,6 +84,13 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro
if err := q.prepare(); err != nil {
return nil, fmt.Errorf("failed to prepare statements: %w", err)
}
+ // A pause kept in the file holds this object from its first pop
+ if paused, err := readPaused(q.pausedAt); err != nil {
+ q.closeStmts()
+ return nil, fmt.Errorf("failed to read pause: %w", err)
+ } else if paused {
+ q.setPaused(true)
+ }
return q, nil
}
@@ -211,7 +219,8 @@ func indexColumns(db *sql.DB, index string) ([]string, error) {
}
// prepare prepares every statement of the queue once. Each read keeps
-// status = 'pending' literal, or SQLite cannot use the partial index.
+// status = 'pending' literal, or SQLite cannot use the partial index. Pops
+// select nothing while the pause row exists.
func (q *Queue) prepare() (err error) {
defer func() {
if err != nil {
@@ -223,7 +232,8 @@ func (q *Queue) prepare() (err error) {
if q.priority {
cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data, job_id"
}
- pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)
+ pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' AND NOT EXISTS (SELECT 1 FROM %s WHERE key = 'paused_at') ORDER BY %s",
+ q.table, q.meta, q.order)
// fin ends a processing row: deleted, or kept with its end as status
fin := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table)
@@ -248,6 +258,7 @@ func (q *Queue) prepare() (err error) {
{&q.count, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", q.table)},
{&q.countCapped, fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE status = 'pending' LIMIT %d)", q.table, remainingCap)},
{&q.values, fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)},
+ {&q.pausedAt, fmt.Sprintf("SELECT value FROM %s WHERE key = 'paused_at'", q.meta)},
{&q.tally, fmt.Sprintf("UPDATE %s SET value = value + 1 WHERE key = ?", q.meta)},
} {
if *s.stmt, err = q.f.db.Prepare(s.query); err != nil {
@@ -260,7 +271,7 @@ func (q *Queue) prepare() (err error) {
// closeStmts closes the statements prepared so far
func (q *Queue) closeStmts() {
- for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.fin, q.count, q.countCapped, q.values, q.putJob, q.unmark, q.hasMark, q.isMarked, q.tally} {
+ for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.fin, q.count, q.countCapped, q.values, q.putJob, q.unmark, q.hasMark, q.isMarked, q.pausedAt, q.tally} {
if s != nil {
s.Close()
}
@@ -371,13 +382,17 @@ func (q *Queue) enqueue(item any, priority int, id string) bool {
}
// claimN moves up to n pending rows to processing in one statement and
-// returns them in queue order
-func (q *Queue) claimN(n int) ([]claimed, bool, error) {
+// returns them in queue order; refused reports a pause stopped it
+func (q *Queue) claimN(n int) (out []claimed, capped, refused bool, err error) {
+ g := q.pz.gen.Load()
q.f.mu.Lock()
- out, capped, err := q.claimGated(n)
+ out, capped, refused, err = q.claimGated(n)
q.f.mu.Unlock()
- if err != nil {
- return nil, false, err
+ if refused {
+ q.refused(g)
+ }
+ if err != nil || refused {
+ return nil, false, refused, err
}
// RETURNING order is unspecified
@@ -387,23 +402,29 @@ func (q *Queue) claimN(n int) ([]claimed, bool, error) {
}
return cmp.Compare(a.id, b.id)
})
- return out, capped, nil
+ return out, capped, false, nil
}
// claimGated claims up to n rows; with a live cancel mark on the file it
// runs in one transaction that drops each marked row it claims (outcome
// cancelled) and claims again, at most maxDrops drops; capped reports that
-// it stopped there. The caller holds the write mutex.
-func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) {
+// it stopped there. An empty claim reads the pause row: refused when it is
+// set. The caller holds the write mutex.
+// ponytail: outside a tx a Pause and Resume from another process, both
+// between the claim and that read, make one empty pop look like a failure.
+func (q *Queue) claimGated(n int) (out []claimed, capped, refused bool, err error) {
now := time.Now()
if !q.anyMark(now) {
- out, err = q.claimRows(q.claim, n)
- return out, false, err
+ if out, err = q.claimRows(q.claim, n); err != nil || len(out) > 0 {
+ return out, false, false, err
+ }
+ refused, err = readPaused(q.pausedAt)
+ return nil, false, refused, err
}
tx, err := q.f.db.Begin()
if err != nil {
- return nil, false, err
+ return nil, false, false, err
}
defer tx.Rollback()
@@ -414,7 +435,7 @@ func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) {
// No more than the drops left: a batch never passes the cap
rows, err := q.claimRows(claim, min(n-len(out), maxDrops-drops))
if err != nil {
- return nil, false, err
+ return nil, false, false, err
}
if len(rows) == 0 {
break
@@ -423,19 +444,26 @@ func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) {
id := jobIDOf(c.once, c.data)
drop, err := marked(isMarked, id, now)
if err != nil {
- return nil, false, err
+ return nil, false, false, err
}
if !drop {
out = append(out, c)
continue
}
if _, err := q.end(tx, c.id, id, contract.JobCancelled, "cancelled", nil, now); err != nil {
- return nil, false, err
+ return nil, false, false, err
}
drops++
}
}
- return out, len(out) < n && drops >= maxDrops, tx.Commit()
+ capped = len(out) < n && drops >= maxDrops
+ if len(out) == 0 && !capped {
+ // The write lock is held: the row read is the one the claim saw
+ if refused, err = readPaused(tx.Stmt(q.pausedAt)); err != nil {
+ return nil, false, false, err
+ }
+ }
+ return out, capped, refused, tx.Commit()
}
// claimRows runs a claim statement and reads every row it returns; the
@@ -471,53 +499,70 @@ func (q *Queue) Dequeue() (any, bool) {
return nil, false
}
+ g := q.pz.gen.Load()
q.f.mu.Lock()
- defer q.f.mu.Unlock()
+ data, ok, refused := q.dequeue()
+ q.f.mu.Unlock()
+ if refused {
+ q.refused(g)
+ }
+ return data, ok
+}
+// dequeue is Dequeue under the write mutex; an empty pop reads the pause
+// row, refused when it is set.
+func (q *Queue) dequeue() (data []byte, ok, refused bool) {
now := time.Now()
if !q.anyMark(now) {
data, _, err := popRow(q.pop)
- return data, err == nil
+ if errors.Is(err, sql.ErrNoRows) {
+ refused, _ = readPaused(q.pausedAt)
+ }
+ return data, err == nil, refused
}
for {
- data, capped, err := q.popGated(now)
- if err != nil || data != nil || !capped {
- return data, err == nil && data != nil
+ data, capped, refused, err := q.popGated(now)
+ if err != nil || data != nil || refused || !capped {
+ return data, err == nil && data != nil, refused
}
}
}
// popGated pops in one transaction, dropping each marked item it pops
// (outcome cancelled), at most maxDrops; capped reports it stopped there
-// with nothing. The caller holds the write mutex.
-func (q *Queue) popGated(now time.Time) (data []byte, capped bool, err error) {
+// with nothing, refused that the pause row stopped it. The caller holds
+// the write mutex.
+func (q *Queue) popGated(now time.Time) (data []byte, capped, refused bool, err error) {
tx, err := q.f.db.Begin()
if err != nil {
- return nil, false, err
+ return nil, false, false, err
}
defer tx.Rollback()
pop, isMarked := tx.Stmt(q.pop), tx.Stmt(q.isMarked)
for range maxDrops {
data, once, err := popRow(pop)
if errors.Is(err, sql.ErrNoRows) {
- return nil, false, tx.Commit() // keep the drops
+ if refused, err = readPaused(tx.Stmt(q.pausedAt)); err != nil {
+ return nil, false, false, err
+ }
+ return nil, false, refused, tx.Commit() // keep the drops
}
if err != nil {
- return nil, false, err
+ return nil, false, false, err
}
id := jobIDOf(once, data)
drop, err := marked(isMarked, id, now)
if err != nil {
- return nil, false, err
+ return nil, false, false, err
}
if !drop {
- return data, false, tx.Commit()
+ return data, false, false, tx.Commit()
}
if err := q.record(tx, id, contract.JobCancelled, "cancelled", nil, now); err != nil {
- return nil, false, err
+ return nil, false, false, err
}
}
- return nil, true, tx.Commit()
+ return nil, true, false, tx.Commit()
}
// popRow runs a pop statement: the deleted row's data and job_id, or
@@ -537,8 +582,8 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) {
// A pop stopped at the drop cap has nothing yet: pop again, it can't
// report items left
for {
- c, capped, err := q.claimN(1)
- if err != nil || len(c) == 0 && !capped {
+ c, capped, refused, err := q.claimN(1)
+ if err != nil || refused || len(c) == 0 && !capped {
return nil, false, ""
}
if len(c) > 0 {
@@ -558,9 +603,9 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini
return 0, 0, nil
}
- c, capped, err := q.claimN(want)
- if err != nil {
- return 0, 0, err
+ c, capped, refused, err := q.claimN(want)
+ if err != nil || refused {
+ return 0, 0, err // paused: nothing left to pop until a Resume
}
// At the drop cap items may wait still: report one, so the caller pops again
left := 0
@@ -698,7 +743,7 @@ func (q *Queue) Values() []any {
}
// Purge removes all items from the queue, with its outcome records, cancel
-// marks and totals (spec 6.6, 6.14)
+// marks and totals; a pause stays (spec 6.6, 6.7, 6.14)
func (q *Queue) Purge() {
q.f.mu.Lock()
defer q.f.mu.Unlock()
diff --git a/queue_test.go b/queue_test.go
index f565c6c..41ed23c 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -1892,3 +1892,211 @@ func TestInspectClosed(t *testing.T) {
t.Error("Expected Jobs to fail on a closed queue")
}
}
+
+func TestImplementsPause(t *testing.T) {
+ var (
+ _ varmq.QueuePauser = (*Queue)(nil)
+ _ varmq.QueuePauser = (*PriorityQueue)(nil)
+ _ varmq.PauseNotifier = (*Queue)(nil)
+ _ varmq.PauseNotifier = (*PriorityQueue)(nil)
+ _ varmq.QueueInspector = (*Queue)(nil)
+ _ varmq.QueueInspector = (*PriorityQueue)(nil)
+ )
+}
+
+// refusedPops checks that every pop kind hands out nothing and reports a
+// pause.
+func refusedPops(t *testing.T, q *Queue) {
+ t.Helper()
+ before := q.PauseRefusals()
+ if _, ok := q.Dequeue(); ok {
+ t.Error("Dequeue handed out an item while paused")
+ }
+ if _, ok, _ := q.DequeueWithAckId(); ok {
+ t.Error("DequeueWithAckId handed out an item while paused")
+ }
+ items, acks := make([]any, 4), make([]string, 4)
+ if n, rem, err := q.DequeueBatchWithAckIds(items, acks); n != 0 || rem != 0 || err != nil {
+ t.Errorf("batch pop while paused: got %d %d %v", n, rem, err)
+ }
+ if got := q.PauseRefusals() - before; got != 3 {
+ t.Errorf("Expected 3 refusals, got %d", got)
+ }
+ if !q.Paused() {
+ t.Error("Expected a refused pop to leave Paused true")
+ }
+}
+
+// Pause stops every pop, keeps its first time, and is kept in the file;
+// Resume lets pops go on. Hooks hear each change once.
+func TestPause(t *testing.T) {
+ for _, marked := range []bool{false, true} {
+ t.Run(fmt.Sprintf("marked=%t", marked), func(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ if marked {
+ q.CancelJob("other") // gated pops
+ }
+ var calls []bool
+ var mu sync.Mutex
+ stop := q.OnPauseChange(func(p bool) {
+ mu.Lock()
+ calls = append(calls, p)
+ mu.Unlock()
+ })
+ defer stop()
+ enqueueN(t, q, 3)
+
+ if q.Paused() {
+ t.Fatal("Expected a new queue to run")
+ }
+ if err := q.Pause(); err != nil {
+ t.Fatal(err)
+ }
+ first := stats(t, q)
+ if !q.Paused() || !first.Paused || first.PausedAt == 0 {
+ t.Fatalf("after Pause: Paused %t, stats %+v", q.Paused(), first)
+ }
+ time.Sleep(2 * time.Millisecond)
+ if err := q.Pause(); err != nil {
+ t.Fatal(err)
+ }
+ if s := stats(t, q); s.PausedAt != first.PausedAt {
+ t.Errorf("Expected a second Pause to keep %d, got %d", first.PausedAt, s.PausedAt)
+ }
+ refusedPops(t, q)
+ if !q.Enqueue([]byte("added while paused")) || q.Len() != 4 {
+ t.Error("Expected adds to land while paused")
+ }
+
+ if err := q.Resume(); err != nil {
+ t.Fatal(err)
+ }
+ if err := q.Resume(); err != nil {
+ t.Fatal(err)
+ }
+ if q.Paused() || stats(t, q).Paused {
+ t.Error("Expected Resume to clear the pause")
+ }
+ if _, ok := q.Dequeue(); !ok {
+ t.Error("Expected pops to go on after Resume")
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if !slices.Equal(calls, []bool{true, false}) {
+ t.Errorf("hooks: got %v", calls)
+ }
+ })
+ }
+}
+
+// waitPaused waits for q.Paused to become want.
+func waitPaused(t *testing.T, q *Queue, want bool) {
+ t.Helper()
+ deadline := time.Now().Add(5 * time.Second)
+ for q.Paused() != want {
+ if time.Now().After(deadline) {
+ t.Fatalf("Paused stayed %t", !want)
+ }
+ time.Sleep(pollEvery / 5)
+ }
+}
+
+// A pause from another process stops this one's pops at once; its Resume
+// reaches this object's Paused and hooks by the poll.
+func TestPauseAcrossProcesses(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ other := otherProcess(t, q)
+ resumed := make(chan struct{}, 1)
+ defer q.OnPauseChange(func(p bool) {
+ if !p {
+ resumed <- struct{}{}
+ }
+ })()
+ enqueueN(t, q, 2)
+
+ if err := other.Pause(); err != nil {
+ t.Fatal(err)
+ }
+ if q.Paused() {
+ t.Fatal("Expected the cache to wait for a pop or the poll")
+ }
+ refusedPops(t, q)
+
+ if err := other.Resume(); err != nil {
+ t.Fatal(err)
+ }
+ waitPaused(t, q, false)
+ select {
+ case <-resumed:
+ case <-time.After(5 * time.Second):
+ t.Fatal("no resume hook")
+ }
+ if _, ok := q.Dequeue(); !ok {
+ t.Error("Expected pops to go on after the other process resumed")
+ }
+}
+
+// A pause is kept in the file: it outlives the process, and Purge keeps it.
+func TestPauseSurvivesReopenAndPurge(t *testing.T) {
+ dir := t.TempDir()
+ qs := mustNew(t, dir)
+ q, _ := qs.NewQueue("q")
+ q.Enqueue([]byte("x"))
+ if err := q.Pause(); err != nil {
+ t.Fatal(err)
+ }
+ q.Purge()
+ q.Enqueue([]byte("y"))
+ qs.Close()
+
+ qs = mustNew(t, dir)
+ defer qs.Close()
+ q, _ = qs.NewQueue("q")
+ if !q.Paused() || !stats(t, q).Paused {
+ t.Fatal("Expected the pause to outlive the process")
+ }
+ refusedPops(t, q)
+ q.Resume()
+ if v, ok := q.Dequeue(); !ok || string(v.([]byte)) != "y" {
+ t.Errorf("got %v %t, want y", v, ok)
+ }
+ q.Close()
+ if q.Pause() == nil || q.Resume() == nil {
+ t.Error("Expected Pause and Resume on a closed queue to fail")
+ }
+}
+
+// varmq: a paused queue holds the worker in every process; a Resume from
+// another process wakes it.
+func TestVarmqPause(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ other := otherProcess(t, q)
+
+ ran := make(chan int, 4)
+ w := varmq.NewWorker(func(j varmq.Job[int]) { ran <- j.Data() })
+ defer w.Stop()
+ handle := w.WithPersistentQueue(q)
+ if err := handle.Pause(); err != nil {
+ t.Fatal(err)
+ }
+ if !handle.IsPaused() {
+ t.Fatal("Expected the handle to report the pause")
+ }
+ handle.Add(1)
+ select {
+ case v := <-ran:
+ t.Fatalf("job %d ran while paused", v)
+ case <-time.After(10 * pollEvery):
+ }
+ if err := other.Resume(); err != nil {
+ t.Fatal(err)
+ }
+ select {
+ case <-ran:
+ case <-time.After(5 * time.Second):
+ t.Fatal("job did not run after the other process resumed")
+ }
+}
diff --git a/registry.go b/registry.go
index 5db083a..2d77f2a 100644
--- a/registry.go
+++ b/registry.go
@@ -186,6 +186,7 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error)
if !f.recovered[key] {
if err := h.queue().requeueNoAck(); err != nil {
+ h.queue().Close() // stops a pause poll
return nil, fmt.Errorf("failed to requeue unacknowledged rows: %w", err)
}
f.recovered[key] = true
From 5f0b799ea37d0220e3b75e425b103a2772a56be0 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:30:04 +0600
Subject: [PATCH 13/22] docs: queue inspection and pause in README, CHANGELOG
---
CHANGELOG.md | 2 ++
README.md | 15 +++++++++++++--
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8961d18..688aed2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error.
- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it by seq every 50 ms (new records only), from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d; the count limit spares records under 1 min old. A polling queue object also sweeps retention and expired marks once a minute.
- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per transaction; a single-item pop goes on past that, a batch pop reports an item left); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended.
+- Queue inspection (`contract.QueueInspector`) on both queues, so varmq's REST API and vartop list sqliteq queues. `QueueID` is `sqliteq::fifo|prio:`, `` the first 8 hex of the SHA-256 of the file's resolved path, `` in lower case. `QueueStats`: counts per row status in one statement (cancelled under failed; completed and failed only with `WithRemoveOnComplete(false)`), totals submitted / completed / failed from new table `__sqliteq_meta` (an insert trigger counts adds; a finish counts in its transaction; `Purge` resets them), and the pause. `Jobs`: pending in hand-out order, processing by ack id, kept completed and failed rows newest first, payload the envelope's `data` verbatim, errors from the outcome record. Attempts are not counted (0 pending, 1 after).
+- Durable queue pause (varmq spec 6.7, 6.8): `Pause`, `Resume`, `Paused`, `PauseRefusals` (`varmq.QueuePauser`) and `OnPauseChange` (`varmq.PauseNotifier`). The flag is a row of `__sqliteq_meta`; every pop statement selects nothing while it exists, in every process on the file; it outlives restarts and `Purge`. A refused pop sets `Paused`; while set, the queue object polls the row every 50 ms to see a `Resume` from another process.
- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks.
### Breaking
diff --git a/README.md b/README.md
index 6058b8b..8df1943 100644
--- a/README.md
+++ b/README.md
@@ -192,6 +192,16 @@ queue := worker.WithPersistentQueue(q)
queue.Add("hello")
```
+### Inspection and pause
+
+Both queue types implement varmq's `QueueInspector`, `QueuePauser` and `PauseNotifier`, so a bound queue shows up in varmq's REST API (`GET /queues`, `GET /queues/{id}/jobs`, `PATCH /queues/{id}/actions/pause|resume`) and in vartop, as a redisq queue does.
+
+- **Queue ID**: `sqliteq::fifo:` or `sqliteq::prio:`. `` is the first 8 hex digits of the SHA-256 of the database file's resolved path, so every process on the file agrees, two files with one queue name never collide, and no path shows. `` is in lower case. sqliteq has no in-memory mode, so every queue has a file. A file reached through two different paths (another mount, say) gets two IDs.
+- **Counts**: rows per `status`, read in one statement. Cancelled rows count as failed. Completed and failed rows exist only with `WithRemoveOnComplete(false)`; by default they are deleted, so those counts are 0. Kept rows have no retention.
+- **Totals**: `submitted` counts stored adds (a repeat `EnqueueOnce` is not one), `completed` and `failed` count finishes (`AcknowledgeCancelled` counts as failed; a waiting job dropped for a cancel counts as neither). They live in `__sqliteq_meta`, start at 0 when this version first opens the table, and `Purge` resets them.
+- **Jobs**: pending in hand-out order, processing by ack id, completed and failed (with cancelled, `state` `cancelled`) newest finish first. `payload` is the varmq envelope's `data`, or the stored item itself (as a JSON string when it is not JSON). A job without an id shows its ack id once handed out. sqliteq does not count attempts: they read 0 while pending and 1 after. A kept row loses its start time when it ends. A failed job's `errors` holds its reason when the job has an id and its outcome record is still kept.
+- **Pause**: `Pause()` stores a `paused_at` row in `__sqliteq_meta`. Every pop selects nothing while it exists, so the pause reaches every process on the file at once, outlives restarts, and survives `Purge`; only `Resume()` ends it. Adds, acks and rejects go on. `Paused()` is this object's cache: a pop refused for the pause sets it, and while it is set the queue polls the row every 50 ms, so a `Resume` from another process wakes this one's workers within that time.
+
sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker whose only source is the sqliteq queue claims several jobs in one statement through `DequeueBatchWithAckIds` (a worker bound to several queues claims one job per call), and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works.
## Storage
@@ -290,7 +300,7 @@ Each queue is a table with these columns:
- `id`: autoincrement primary key. Items come out in `id` order, and a priority queue's in `priority`, then `id` order. The ack id `DequeueWithAckId` returns is this id in decimal.
- `priority`: priority queues only; a lower number comes out first. Opening an older table as a priority queue adds it.
- `data`: the stored item.
-- `status`: `pending`, `processing` or `completed`.
+- `status`: `pending`, `processing`, or on a kept row `completed`, `failed` or `cancelled`.
- `ack`: 0 when an item is added, 1 once it is acknowledged or rejected and kept with `WithRemoveOnComplete(false)` (`status` `completed` or `failed`). On open, only `processing` rows with `ack` 0 go back to pending.
- `ack_id`: kept from v0.2 so old tables need no rebuild; no longer used.
- `job_id`: the id an item was added with through `EnqueueOnce`, unique among the rows that have one; NULL for `Enqueue`. Opening an older table adds it.
@@ -299,7 +309,8 @@ Each queue is a table with these columns:
On top of that table:
- Two partial indexes: one covers the pending items in queue order, and a unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on.
-- A claim or a pop is one `UPDATE ... RETURNING` or `DELETE ... RETURNING` statement.
+- A claim or a pop is one `UPDATE ... RETURNING` or `DELETE ... RETURNING` statement. It selects nothing while the queue is paused.
+- `__sqliteq_meta` holds the totals and the pause; a trigger counts each stored add.
- Writes to one file (adds, claims, acks, purges) take one lock in the process, so they wait for each other in Go rather than in SQLite's busy handler. `Len` and `Values` do not take it.
- Concurrent `Enqueue` and `Acknowledge` calls on one file are committed together in one transaction. Each call still gets its own result, and if the shared transaction fails, each call is retried on its own so one bad call fails alone.
From 87077696d6de4b90524f512bee6589d5ca73e3ba Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:34:30 +0600
Subject: [PATCH 14/22] fix: seed an older table's totals from its rows
---
CHANGELOG.md | 2 +-
README.md | 2 +-
inspect.go | 10 +++++++---
queue_test.go | 17 +++++++++++++++++
4 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 688aed2..083136d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error.
- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it by seq every 50 ms (new records only), from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d; the count limit spares records under 1 min old. A polling queue object also sweeps retention and expired marks once a minute.
- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per transaction; a single-item pop goes on past that, a batch pop reports an item left); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended.
-- Queue inspection (`contract.QueueInspector`) on both queues, so varmq's REST API and vartop list sqliteq queues. `QueueID` is `sqliteq::fifo|prio:`, `` the first 8 hex of the SHA-256 of the file's resolved path, `` in lower case. `QueueStats`: counts per row status in one statement (cancelled under failed; completed and failed only with `WithRemoveOnComplete(false)`), totals submitted / completed / failed from new table `__sqliteq_meta` (an insert trigger counts adds; a finish counts in its transaction; `Purge` resets them), and the pause. `Jobs`: pending in hand-out order, processing by ack id, kept completed and failed rows newest first, payload the envelope's `data` verbatim, errors from the outcome record. Attempts are not counted (0 pending, 1 after).
+- Queue inspection (`contract.QueueInspector`) on both queues, so varmq's REST API and vartop list sqliteq queues. `QueueID` is `sqliteq::fifo|prio:`, `` the first 8 hex of the SHA-256 of the file's resolved path, `` in lower case. `QueueStats`: counts per row status in one statement (cancelled under failed; completed and failed only with `WithRemoveOnComplete(false)`), totals submitted / completed / failed from new table `__sqliteq_meta` (an insert trigger counts adds; a finish counts in its transaction; an older table starts from its rows; `Purge` resets them), and the pause. `Jobs`: pending in hand-out order, processing by ack id, kept completed and failed rows newest first, payload the envelope's `data` verbatim, errors from the outcome record. Attempts are not counted (0 pending, 1 after).
- Durable queue pause (varmq spec 6.7, 6.8): `Pause`, `Resume`, `Paused`, `PauseRefusals` (`varmq.QueuePauser`) and `OnPauseChange` (`varmq.PauseNotifier`). The flag is a row of `__sqliteq_meta`; every pop statement selects nothing while it exists, in every process on the file; it outlives restarts and `Purge`. A refused pop sets `Paused`; while set, the queue object polls the row every 50 ms to see a `Resume` from another process.
- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks.
diff --git a/README.md b/README.md
index 8df1943..2c7d256 100644
--- a/README.md
+++ b/README.md
@@ -198,7 +198,7 @@ Both queue types implement varmq's `QueueInspector`, `QueuePauser` and `PauseNot
- **Queue ID**: `sqliteq::fifo:` or `sqliteq::prio:`. `` is the first 8 hex digits of the SHA-256 of the database file's resolved path, so every process on the file agrees, two files with one queue name never collide, and no path shows. `` is in lower case. sqliteq has no in-memory mode, so every queue has a file. A file reached through two different paths (another mount, say) gets two IDs.
- **Counts**: rows per `status`, read in one statement. Cancelled rows count as failed. Completed and failed rows exist only with `WithRemoveOnComplete(false)`; by default they are deleted, so those counts are 0. Kept rows have no retention.
-- **Totals**: `submitted` counts stored adds (a repeat `EnqueueOnce` is not one), `completed` and `failed` count finishes (`AcknowledgeCancelled` counts as failed; a waiting job dropped for a cancel counts as neither). They live in `__sqliteq_meta`, start at 0 when this version first opens the table, and `Purge` resets them.
+- **Totals**: `submitted` counts stored adds (a repeat `EnqueueOnce` is not one), `completed` and `failed` count finishes (`AcknowledgeCancelled` counts as failed; a waiting job dropped for a cancel counts as neither). They live in `__sqliteq_meta`, start from the rows an older table holds when this version first opens it, and `Purge` resets them.
- **Jobs**: pending in hand-out order, processing by ack id, completed and failed (with cancelled, `state` `cancelled`) newest finish first. `payload` is the varmq envelope's `data`, or the stored item itself (as a JSON string when it is not JSON). A job without an id shows its ack id once handed out. sqliteq does not count attempts: they read 0 while pending and 1 after. A kept row loses its start time when it ends. A failed job's `errors` holds its reason when the job has an id and its outcome record is still kept.
- **Pause**: `Pause()` stores a `paused_at` row in `__sqliteq_meta`. Every pop selects nothing while it exists, so the pause reaches every process on the file at once, outlives restarts, and survives `Purge`; only `Resume()` ends it. Adds, acks and rejects go on. `Paused()` is this object's cache: a pop refused for the pause sets it, and while it is set the queue polls the row every 50 ms, so a `Resume` from another process wakes this one's workers within that time.
diff --git a/inspect.go b/inspect.go
index b38d4aa..622e3b8 100644
--- a/inspect.go
+++ b/inspect.go
@@ -34,7 +34,11 @@ const metaSuffix = "__sqliteq_meta"
func (q *Queue) initMeta() error {
for _, s := range []string{
`CREATE TABLE IF NOT EXISTS %[2]s (key TEXT PRIMARY KEY, value INTEGER NOT NULL)`,
- `INSERT OR IGNORE INTO %[2]s (key, value) VALUES ('submitted', 0), ('completed', 0), ('failed', 0)`,
+ // An older table starts from the rows it holds
+ `INSERT OR IGNORE INTO %[2]s (key, value) VALUES
+ ('submitted', (SELECT COUNT(*) FROM %[1]s)),
+ ('completed', (SELECT COUNT(*) FROM %[1]s WHERE status = 'completed')),
+ ('failed', (SELECT COUNT(*) FROM %[1]s WHERE status IN ('failed', 'cancelled')))`,
`CREATE TRIGGER IF NOT EXISTS %[3]s AFTER INSERT ON %[1]s BEGIN
UPDATE %[2]s SET value = value + 1 WHERE key = 'submitted';
END`,
@@ -68,8 +72,8 @@ func (q *Queue) QueueID() string {
// QueueStats implements contract.QueueInspector, in one statement (one
// snapshot). Counts: rows per status, cancelled under failed. Totals:
// adds stored and jobs finished (cancelled ones as failed; a waiting job
-// dropped for a cancel is neither) since the table was first opened by
-// this version or the last Purge. Paused is the file's, not the cache.
+// dropped for a cancel is neither) since the last Purge; an older table
+// starts from the rows it held. Paused is the file's, not the cache.
// ponytail: counts scan the table; index status if kept rows grow large.
func (q *Queue) QueueStats(ctx context.Context) (contract.QueueStats, error) {
if q.closed.Load() {
diff --git a/queue_test.go b/queue_test.go
index 41ed23c..3a9d01b 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -2100,3 +2100,20 @@ func TestVarmqPause(t *testing.T) {
t.Fatal("job did not run after the other process resumed")
}
}
+
+// An older table's totals start from the rows it holds.
+func TestTotalsSeededFromRows(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "old.db")
+ writeV023File(t, path, "q", false)
+ qs := mustNew(t, path, WithSingleFile())
+ defer qs.Close()
+ q, err := qs.NewQueue("q")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var rows int64
+ q.f.db.QueryRow(`SELECT COUNT(*) FROM "q"`).Scan(&rows)
+ if s := stats(t, q); rows == 0 || s.Totals.Submitted != uint64(rows) {
+ t.Errorf("got %+v, want submitted %d", s.Totals, rows)
+ }
+}
From f0f82b58b1121b40f97c438ad68e98e76c8e5763 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:48:58 +0600
Subject: [PATCH 15/22] fix: pause cache gen check and swap in one step
observe checked gen, then set the cache: a Pause or Resume of the same
object between the two was overwritten by a stale poll or refusal read
(cache false while the row exists, poll stopped). setPaused now checks
gen and swaps under pz.mu; poll start and hooks run after unlock.
---
pause.go | 52 ++++++++++++++++++++++++++-------------------------
queue.go | 2 +-
queue_test.go | 28 +++++++++++++++++++++++++++
3 files changed, 56 insertions(+), 26 deletions(-)
diff --git a/pause.go b/pause.go
index 30451af..b192d34 100644
--- a/pause.go
+++ b/pause.go
@@ -18,10 +18,12 @@ import (
// the poll, which reads the row every pollEvery, so a Resume from another
// process reaches it within pollEvery.
-// pauseState is a Queue's Paused cache and OnPauseChange hooks.
+// pauseState is a Queue's Paused cache and OnPauseChange hooks. mu makes
+// each gen check and cache swap one step, so a stale read never overwrites
+// a newer Pause or Resume of this object.
type pauseState struct {
- paused atomic.Bool
- gen atomic.Uint64 // moves at each Pause and Resume
+ paused atomic.Bool // written under mu
+ gen atomic.Uint64 // moves at each Pause and Resume; under mu
refusals atomic.Uint64 // pops refused for a pause
mu sync.Mutex
hooks map[*func(bool)]struct{}
@@ -51,8 +53,7 @@ func (q *Queue) setPauseRow(p bool, query string, args ...any) error {
if err != nil {
return err
}
- q.pz.gen.Add(1)
- q.setPaused(p)
+ q.setPaused(p, true, 0)
return nil
}
@@ -90,12 +91,28 @@ func (q *Queue) OnPauseChange(fn func(paused bool)) (cancel func()) {
}
}
-// setPaused updates the cache; on a change it starts the poll (paused) and
-// calls the hooks.
-func (q *Queue) setPaused(p bool) {
+// setPaused sets the cache to p: own for this object's Pause or Resume,
+// which moves gen; else only if gen is still g, the gen of the call that
+// read p (a Pause or Resume since may be newer than p). On a change, after
+// mu, it starts the poll (paused) and calls the hooks.
+func (q *Queue) setPaused(p, own bool, g uint64) {
+ q.pz.mu.Lock()
+ if own {
+ q.pz.gen.Add(1)
+ } else if q.pz.gen.Load() != g {
+ q.pz.mu.Unlock()
+ return
+ }
if q.pz.paused.Swap(p) == p {
+ q.pz.mu.Unlock()
return
}
+ fns := make([]func(bool), 0, len(q.pz.hooks))
+ for h := range q.pz.hooks {
+ fns = append(fns, *h)
+ }
+ q.pz.mu.Unlock()
+
if p {
w := &q.w
w.mu.Lock()
@@ -104,37 +121,22 @@ func (q *Queue) setPaused(p bool) {
}
w.mu.Unlock()
}
-
- q.pz.mu.Lock()
- fns := make([]func(bool), 0, len(q.pz.hooks))
- for h := range q.pz.hooks {
- fns = append(fns, *h)
- }
- q.pz.mu.Unlock()
for _, fn := range fns {
fn(p)
}
}
-// observe applies p, read by a call that began at gen g, unless a Pause or
-// Resume of this object came since: p may then be older than what it set.
-func (q *Queue) observe(g uint64, p bool) {
- if q.pz.gen.Load() == g {
- q.setPaused(p)
- }
-}
-
// refused records a pop refused for a pause, by a call that began at gen g.
func (q *Queue) refused(g uint64) {
q.pz.refusals.Add(1)
- q.observe(g, true)
+ q.setPaused(true, false, g)
}
// pollPause reads the pause row for the poll.
func (q *Queue) pollPause() {
g := q.pz.gen.Load()
if paused, err := readPaused(q.pausedAt); err == nil {
- q.observe(g, paused)
+ q.setPaused(paused, false, g)
}
}
diff --git a/queue.go b/queue.go
index 6f3aa70..985d14e 100644
--- a/queue.go
+++ b/queue.go
@@ -89,7 +89,7 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro
q.closeStmts()
return nil, fmt.Errorf("failed to read pause: %w", err)
} else if paused {
- q.setPaused(true)
+ q.setPaused(true, true, 0)
}
return q, nil
diff --git a/queue_test.go b/queue_test.go
index 3a9d01b..b5d013a 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -2117,3 +2117,31 @@ func TestTotalsSeededFromRows(t *testing.T) {
t.Errorf("got %+v, want submitted %d", s.Totals, rows)
}
}
+
+// A stopped hook gets no calls; a refusal read before this object's own
+// Resume does not undo it.
+func TestPauseHookStopAndStaleRefusal(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ var calls []bool
+ var mu sync.Mutex
+ defer q.OnPauseChange(func(p bool) {
+ mu.Lock()
+ calls = append(calls, p)
+ mu.Unlock()
+ })()
+ q.OnPauseChange(func(bool) { t.Error("A stopped hook was called") })()
+
+ q.Pause()
+ g := q.pz.gen.Load()
+ q.Resume()
+ q.refused(g)
+ if q.Paused() {
+ t.Error("Expected a stale refusal to leave the Resume")
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if !slices.Equal(calls, []bool{true, false}) {
+ t.Errorf("hooks: got %v", calls)
+ }
+}
From 7165e94541ce16ba73504aa9380f18ae63300aec Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:49:24 +0600
Subject: [PATCH 16/22] fix: pop again when a Resume lands after an empty pop
Outside a tx the empty claim or pop and the pause row read are two
statements; a Resume from another process between them made a refused
pop look unrefused, so varmq raised ErrFailedToDequeue. An empty pop
found unpaused now pops once more.
---
queue.go | 40 ++++++++++++++++++++++++++++------------
queue_test.go | 35 +++++++++++++++++++++++++++++++++++
2 files changed, 63 insertions(+), 12 deletions(-)
diff --git a/queue.go b/queue.go
index 985d14e..c3d820a 100644
--- a/queue.go
+++ b/queue.go
@@ -19,6 +19,10 @@ import (
// remainingCap bounds the pending count a batch pop reports
const remainingCap = 256
+// afterEmptyPop runs between an empty pop outside a tx and its read of the
+// pause row; tests only.
+var afterEmptyPop = func() {}
+
// Queue implements the Queue interface using SQLite as the storage backend
type Queue struct {
f *file
@@ -409,17 +413,23 @@ func (q *Queue) claimN(n int) (out []claimed, capped, refused bool, err error) {
// runs in one transaction that drops each marked row it claims (outcome
// cancelled) and claims again, at most maxDrops drops; capped reports that
// it stopped there. An empty claim reads the pause row: refused when it is
-// set. The caller holds the write mutex.
-// ponytail: outside a tx a Pause and Resume from another process, both
-// between the claim and that read, make one empty pop look like a failure.
+// set. Outside a tx a Resume from another process can land between the two,
+// so an empty claim found unpaused claims once more. The caller holds the
+// write mutex.
+// ponytail: a second Pause and Resume, both in the window again, still make
+// one empty pop look like a failure.
func (q *Queue) claimGated(n int) (out []claimed, capped, refused bool, err error) {
now := time.Now()
if !q.anyMark(now) {
- if out, err = q.claimRows(q.claim, n); err != nil || len(out) > 0 {
- return out, false, false, err
+ for try := range 2 {
+ if out, err = q.claimRows(q.claim, n); err != nil || len(out) > 0 {
+ return out, false, false, err
+ }
+ afterEmptyPop()
+ if refused, err = readPaused(q.pausedAt); err != nil || refused || try == 1 {
+ return nil, false, refused, err
+ }
}
- refused, err = readPaused(q.pausedAt)
- return nil, false, refused, err
}
tx, err := q.f.db.Begin()
@@ -510,15 +520,21 @@ func (q *Queue) Dequeue() (any, bool) {
}
// dequeue is Dequeue under the write mutex; an empty pop reads the pause
-// row, refused when it is set.
+// row, refused when it is set, and pops once more when it is not (see
+// claimGated).
func (q *Queue) dequeue() (data []byte, ok, refused bool) {
now := time.Now()
if !q.anyMark(now) {
- data, _, err := popRow(q.pop)
- if errors.Is(err, sql.ErrNoRows) {
- refused, _ = readPaused(q.pausedAt)
+ for try := range 2 {
+ data, _, err := popRow(q.pop)
+ if !errors.Is(err, sql.ErrNoRows) {
+ return data, err == nil, false
+ }
+ afterEmptyPop()
+ if refused, err = readPaused(q.pausedAt); err != nil || refused || try == 1 {
+ return nil, false, refused
+ }
}
- return data, err == nil, refused
}
for {
data, capped, refused, err := q.popGated(now)
diff --git a/queue_test.go b/queue_test.go
index b5d013a..7e3ea59 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -2145,3 +2145,38 @@ func TestPauseHookStopAndStaleRefusal(t *testing.T) {
t.Errorf("hooks: got %v", calls)
}
}
+
+// A Resume from another process between an empty pop and its read of the
+// pause row is not a failed pop: the pop goes again and hands out the item.
+func TestResumeBetweenPopAndRead(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ other := otherProcess(t, q)
+ t.Cleanup(func() { afterEmptyPop = func() {} })
+ pops := []struct {
+ name string
+ pop func() bool
+ }{
+ {"Dequeue", func() bool { _, ok := q.Dequeue(); return ok }},
+ {"DequeueWithAckId", func() bool { _, ok, _ := q.DequeueWithAckId(); return ok }},
+ {"DequeueBatchWithAckIds", func() bool {
+ n, _, _ := q.DequeueBatchWithAckIds(make([]any, 2), make([]string, 2))
+ return n == 1
+ }},
+ }
+ for _, p := range pops {
+ q.Enqueue([]byte(p.name))
+ if err := other.Pause(); err != nil {
+ t.Fatal(err)
+ }
+ afterEmptyPop = func() {
+ afterEmptyPop = func() {}
+ if err := other.Resume(); err != nil {
+ t.Error(err)
+ }
+ }
+ if !p.pop() {
+ t.Errorf("%s: Expected the item after the Resume", p.name)
+ }
+ }
+}
From 75726d94ff90c82859dc4914819d35b4730c7d7a Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:49:39 +0600
Subject: [PATCH 17/22] feat: Jobs pages cancelled rows (openapi 1.4.0)
---
inspect.go | 10 ++++++----
queue_test.go | 32 ++++++++++++++++++++++++++++++++
2 files changed, 38 insertions(+), 4 deletions(-)
diff --git a/inspect.go b/inspect.go
index 622e3b8..f753d87 100644
--- a/inspect.go
+++ b/inspect.go
@@ -103,9 +103,9 @@ func (q *Queue) QueueStats(ctx context.Context) (contract.QueueStats, error) {
}
// Jobs implements contract.QueueInspector: pending in hand-out order,
-// processing by ack id, completed and failed (with cancelled, state
-// cancelled) newest finish first. Not counted by sqliteq: attempts are 0
-// pending, else 1; an ended row's start is lost. errors: one entry, from
+// processing by ack id, completed, failed (with cancelled, state
+// cancelled) and cancelled newest finish first. Not counted by sqliteq:
+// attempts are 0 pending, else 1; an ended row's start is lost. errors: one entry, from
// the job's outcome record, when it has one.
// ponytail: ended pages sort by updated_at unindexed; fine until kept rows
// grow large.
@@ -120,6 +120,8 @@ func (q *Queue) Jobs(ctx context.Context, state contract.JobState, offset, limit
where, order = "status = 'completed'", "updated_at DESC, id DESC"
case contract.JobFailed:
where, order = "status IN ('failed', 'cancelled')", "updated_at DESC, id DESC"
+ case contract.JobCancelled:
+ where, order = "status = 'cancelled'", "updated_at DESC, id DESC"
default:
return nil, fmt.Errorf("sqliteq: unknown job state %q", state)
}
@@ -161,7 +163,7 @@ func (q *Queue) Jobs(ctx context.Context, state contract.JobState, offset, limit
j.StartedAt, j.Attempts = unixMilli(updated), 1
default:
j.FinishedAt, j.Attempts = unixMilli(updated), 1
- if j.ID != "" && state == contract.JobFailed {
+ if j.ID != "" && state != contract.JobCompleted {
ended = append(ended, len(jobs))
}
}
diff --git a/queue_test.go b/queue_test.go
index 7e3ea59..96a5da2 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -2180,3 +2180,35 @@ func TestResumeBetweenPopAndRead(t *testing.T) {
}
}
}
+
+// Completed, failed and cancelled pages sort by finish, not by add.
+func TestJobsNewestFinishFirst(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(false))
+ defer qs.Close()
+ for _, id := range []string{"a", "b", "c", "d", "e", "f"} {
+ q.Enqueue(job(id))
+ }
+ acks := make([]string, 6)
+ for i := range acks {
+ _, acks[i] = pop(t, q)
+ }
+ q.Acknowledge(acks[1])
+ q.Acknowledge(acks[0])
+ q.Reject(acks[3], "x")
+ q.Reject(acks[2], "x")
+ q.AcknowledgeCancelled(acks[5], "x")
+ q.AcknowledgeCancelled(acks[4], "y")
+ if got := ids(jobs(t, q, contract.JobCompleted, 0, 10)); !slices.Equal(got, []string{"a", "b"}) {
+ t.Errorf("completed: got %q", got)
+ }
+ if got := ids(jobs(t, q, contract.JobFailed, 0, 10)); !slices.Equal(got, []string{"e", "f", "c", "d"}) {
+ t.Errorf("failed: got %q", got)
+ }
+ cancelled := jobs(t, q, contract.JobCancelled, 0, 10)
+ if got := ids(cancelled); !slices.Equal(got, []string{"e", "f"}) {
+ t.Fatalf("cancelled: got %q", got)
+ }
+ if e := cancelled[0]; e.State != contract.JobCancelled || len(e.Errors) != 1 || e.Errors[0].Error != "y" {
+ t.Errorf("cancelled job: got %+v", e)
+ }
+}
From 31b2c2235143a36a5519b03408c7f4803e154c9b Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:50:21 +0600
Subject: [PATCH 18/22] test: totals on no-op finishes and seeded ended rows,
file pause in stats, Purge keeps paused_at
---
queue_test.go | 58 +++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 52 insertions(+), 6 deletions(-)
diff --git a/queue_test.go b/queue_test.go
index 96a5da2..1b564e9 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -1862,6 +1862,9 @@ func TestJobs(t *testing.T) {
if j := jobs(t, q, contract.JobPending, 0, 0); j == nil || len(j) != 0 {
t.Errorf("limit 0: got %#v", j)
}
+ if j := jobs(t, q, contract.JobPending, 0, -1); j == nil || len(j) != 0 {
+ t.Errorf("limit -1: got %#v", j)
+ }
if _, err := q.Jobs(context.Background(), "bogus", 0, 10); err == nil {
t.Error("Expected an unknown state to fail")
}
@@ -1895,12 +1898,10 @@ func TestInspectClosed(t *testing.T) {
func TestImplementsPause(t *testing.T) {
var (
- _ varmq.QueuePauser = (*Queue)(nil)
- _ varmq.QueuePauser = (*PriorityQueue)(nil)
- _ varmq.PauseNotifier = (*Queue)(nil)
- _ varmq.PauseNotifier = (*PriorityQueue)(nil)
- _ varmq.QueueInspector = (*Queue)(nil)
- _ varmq.QueueInspector = (*PriorityQueue)(nil)
+ _ varmq.QueuePauser = (*Queue)(nil)
+ _ varmq.QueuePauser = (*PriorityQueue)(nil)
+ _ varmq.PauseNotifier = (*Queue)(nil)
+ _ varmq.PauseNotifier = (*PriorityQueue)(nil)
)
}
@@ -2022,6 +2023,9 @@ func TestPauseAcrossProcesses(t *testing.T) {
if q.Paused() {
t.Fatal("Expected the cache to wait for a pop or the poll")
}
+ if !stats(t, q).Paused {
+ t.Error("Expected QueueStats to read the pause from the file")
+ }
refusedPops(t, q)
if err := other.Resume(); err != nil {
@@ -2047,7 +2051,11 @@ func TestPauseSurvivesReopenAndPurge(t *testing.T) {
if err := q.Pause(); err != nil {
t.Fatal(err)
}
+ at := stats(t, q).PausedAt
q.Purge()
+ if s := stats(t, q); s.PausedAt != at || s.Totals != (contract.JobTotals{}) {
+ t.Errorf("after Purge: got %+v, want paused_at %d and no totals", s, at)
+ }
q.Enqueue([]byte("y"))
qs.Close()
@@ -2212,3 +2220,41 @@ func TestJobsNewestFinishFirst(t *testing.T) {
t.Errorf("cancelled job: got %+v", e)
}
}
+
+// Seeding counts kept ended rows too: cancelled as failed.
+func TestTotalsSeededFromEndedRows(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(false))
+ defer qs.Close()
+ enqueueN(t, q, 4)
+ _, a := pop(t, q)
+ _, b := pop(t, q)
+ _, c := pop(t, q)
+ q.Acknowledge(a)
+ q.Reject(b, "boom")
+ q.AcknowledgeCancelled(c, "stop")
+ if _, err := q.f.db.Exec("DROP TABLE " + q.meta); err != nil {
+ t.Fatal(err)
+ }
+ want := contract.JobTotals{Submitted: 4, Completed: 1, Failed: 2}
+ if got := stats(t, otherProcess(t, q)).Totals; got != want {
+ t.Errorf("got %+v, want %+v", got, want)
+ }
+}
+
+// A finish that ends no row (repeat, stale or unknown ack) counts nothing.
+func TestTotalsIgnoreNoopFinish(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("a"))
+ _, ack := pop(t, q)
+ if !q.Acknowledge(ack) {
+ t.Fatal("Acknowledge failed")
+ }
+ if q.Acknowledge(ack) || q.Reject(ack, "x") || q.AcknowledgeCancelled(ack, "x") ||
+ q.Acknowledge("garbage") || q.Reject("999", "x") {
+ t.Error("Expected a finish of no processing row to return false")
+ }
+ if got, want := stats(t, q).Totals, (contract.JobTotals{Submitted: 1, Completed: 1}); got != want {
+ t.Errorf("got %+v, want %+v", got, want)
+ }
+}
From 42fb1ee3a1fda3f8577ac75cf75ab775fb96e61f Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 05:50:21 +0600
Subject: [PATCH 19/22] docs: cancelled page, seeded totals gap, processes
sharing a file
---
CHANGELOG.md | 2 +-
README.md | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 083136d..467165e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error.
- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it by seq every 50 ms (new records only), from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d; the count limit spares records under 1 min old. A polling queue object also sweeps retention and expired marks once a minute.
- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per transaction; a single-item pop goes on past that, a batch pop reports an item left); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended.
-- Queue inspection (`contract.QueueInspector`) on both queues, so varmq's REST API and vartop list sqliteq queues. `QueueID` is `sqliteq::fifo|prio:`, `` the first 8 hex of the SHA-256 of the file's resolved path, `` in lower case. `QueueStats`: counts per row status in one statement (cancelled under failed; completed and failed only with `WithRemoveOnComplete(false)`), totals submitted / completed / failed from new table `__sqliteq_meta` (an insert trigger counts adds; a finish counts in its transaction; an older table starts from its rows; `Purge` resets them), and the pause. `Jobs`: pending in hand-out order, processing by ack id, kept completed and failed rows newest first, payload the envelope's `data` verbatim, errors from the outcome record. Attempts are not counted (0 pending, 1 after).
+- Queue inspection (`contract.QueueInspector`) on both queues, so varmq's REST API and vartop list sqliteq queues. `QueueID` is `sqliteq::fifo|prio:`, `` the first 8 hex of the SHA-256 of the file's resolved path, `` in lower case. `QueueStats`: counts per row status in one statement (cancelled under failed; completed and failed only with `WithRemoveOnComplete(false)`), totals submitted / completed / failed from new table `__sqliteq_meta` (an insert trigger counts adds; a finish counts in its transaction; an older table starts from its rows, so jobs deleted before the upgrade are missing; `Purge` resets them), and the pause. `Jobs`: pending in hand-out order, processing by ack id, kept completed, failed (with cancelled) and cancelled rows newest first, payload the envelope's `data` verbatim, errors from the outcome record. Attempts are not counted (0 pending, 1 after).
- Durable queue pause (varmq spec 6.7, 6.8): `Pause`, `Resume`, `Paused`, `PauseRefusals` (`varmq.QueuePauser`) and `OnPauseChange` (`varmq.PauseNotifier`). The flag is a row of `__sqliteq_meta`; every pop statement selects nothing while it exists, in every process on the file; it outlives restarts and `Purge`. A refused pop sets `Paused`; while set, the queue object polls the row every 50 ms to see a `Resume` from another process.
- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks.
diff --git a/README.md b/README.md
index 2c7d256..09e9ab7 100644
--- a/README.md
+++ b/README.md
@@ -198,8 +198,8 @@ Both queue types implement varmq's `QueueInspector`, `QueuePauser` and `PauseNot
- **Queue ID**: `sqliteq::fifo:` or `sqliteq::prio:`. `` is the first 8 hex digits of the SHA-256 of the database file's resolved path, so every process on the file agrees, two files with one queue name never collide, and no path shows. `` is in lower case. sqliteq has no in-memory mode, so every queue has a file. A file reached through two different paths (another mount, say) gets two IDs.
- **Counts**: rows per `status`, read in one statement. Cancelled rows count as failed. Completed and failed rows exist only with `WithRemoveOnComplete(false)`; by default they are deleted, so those counts are 0. Kept rows have no retention.
-- **Totals**: `submitted` counts stored adds (a repeat `EnqueueOnce` is not one), `completed` and `failed` count finishes (`AcknowledgeCancelled` counts as failed; a waiting job dropped for a cancel counts as neither). They live in `__sqliteq_meta`, start from the rows an older table holds when this version first opens it, and `Purge` resets them.
-- **Jobs**: pending in hand-out order, processing by ack id, completed and failed (with cancelled, `state` `cancelled`) newest finish first. `payload` is the varmq envelope's `data`, or the stored item itself (as a JSON string when it is not JSON). A job without an id shows its ack id once handed out. sqliteq does not count attempts: they read 0 while pending and 1 after. A kept row loses its start time when it ends. A failed job's `errors` holds its reason when the job has an id and its outcome record is still kept.
+- **Totals**: `submitted` counts stored adds (a repeat `EnqueueOnce` is not one), `completed` and `failed` count finishes (`AcknowledgeCancelled` counts as failed; a waiting job dropped for a cancel counts as neither). They live in `__sqliteq_meta`, start from the rows an older table holds when this version first opens it, and `Purge` resets them. Seeded totals count the rows the table holds then, so jobs deleted before the upgrade (the default `WithRemoveOnComplete(true)`) are missing from them.
+- **Jobs**: pending in hand-out order, processing by ack id, completed, failed (with cancelled, `state` `cancelled`) and cancelled newest finish first. `payload` is the varmq envelope's `data`, or the stored item itself (as a JSON string when it is not JSON). A job without an id shows its ack id once handed out. sqliteq does not count attempts: they read 0 while pending and 1 after. A kept row loses its start time when it ends. A failed job's `errors` holds its reason when the job has an id and its outcome record is still kept.
- **Pause**: `Pause()` stores a `paused_at` row in `__sqliteq_meta`. Every pop selects nothing while it exists, so the pause reaches every process on the file at once, outlives restarts, and survives `Purge`; only `Resume()` ends it. Adds, acks and rejects go on. `Paused()` is this object's cache: a pop refused for the pause sets it, and while it is set the queue polls the row every 50 ms, so a `Resume` from another process wakes this one's workers within that time.
sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker whose only source is the sqliteq queue claims several jobs in one statement through `DequeueBatchWithAckIds` (a worker bound to several queues claims one job per call), and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works.
@@ -238,11 +238,11 @@ In the folder layout the queue name becomes a file name, so it may only contain
Names are case-insensitive in both layouts: `Jobs` and `jobs` are the same queue, stored in `jobs.db` in the folder layout.
-### One process per file
+### Processes and files
-Run one process per database file. sqliteq keeps a registry in the process, keyed by the file's real path (made absolute, with symlinks resolved). Every `New` in the process that reaches the same file shares one connection pool and one write lock, so two `New` calls on one path never write through two competing pools. Two `Queues` values can therefore hold the same queue, and you get the same handle from both. The file closes when the last `Queues` using it is closed.
+sqliteq keeps a registry in the process, keyed by the file's real path (made absolute, with symlinks resolved). Every `New` in the process that reaches the same file shares one connection pool and one write lock, so two `New` calls on one path never write through two competing pools. Two `Queues` values can therefore hold the same queue, and you get the same handle from both. The file closes when the last `Queues` using it is closed.
-Another process is not coordinated with this one. The first time a process opens a queue, it puts every item still marked as processing back to pending, so a second process opening a file that is in use would rerun the first process's running jobs. sqliteq does not take an OS lock to stop that.
+Other processes can open the same file. SQLite's lock orders their writes, and what lives in the file reaches every process: a pause stops every process's pops at once, and a `Resume`, a cancel or a job outcome reaches the others within 50 ms (they poll). Recovery is the limit: the first time a process opens a queue, it puts every item still marked as processing back to pending, so a process that opens a queue while another one runs its jobs reruns those jobs. sqliteq does not take an OS lock to stop that. Open the queue in every process before any of them takes jobs, or let only one process consume it; producers, observers and `Pause`/`Resume` callers do not take jobs.
On macOS, spell a path the same way every time: the registry key keeps the letter case you pass, so `Data` and `data` would be two entries for one folder. On Windows, Go resolves existing folders and files to their on-disk letter case, so different spellings of them share one entry.
From 734eccbe53b0405951220a8a0d1d90114abec21e Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 06:01:40 +0600
Subject: [PATCH 20/22] test: stale refusal racing Resume; no-op finish totals
with kept rows
---
queue_test.go | 51 +++++++++++++++++++++++++++++++++++++--------------
1 file changed, 37 insertions(+), 14 deletions(-)
diff --git a/queue_test.go b/queue_test.go
index 1b564e9..a8ab137 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -2148,10 +2148,29 @@ func TestPauseHookStopAndStaleRefusal(t *testing.T) {
t.Error("Expected a stale refusal to leave the Resume")
}
mu.Lock()
- defer mu.Unlock()
if !slices.Equal(calls, []bool{true, false}) {
t.Errorf("hooks: got %v", calls)
}
+ mu.Unlock()
+
+ // Racing: a refusal whose gen check passes must not apply after a
+ // whole Resume ran
+ q2, qs2 := openQueue(t)
+ defer qs2.Close()
+ for i := range 20000 {
+ q2.Pause()
+ g := q2.pz.gen.Load()
+ done := make(chan struct{})
+ go func() {
+ q2.refused(g)
+ close(done)
+ }()
+ q2.Resume()
+ <-done
+ if q2.Paused() {
+ t.Fatalf("round %d: a stale refusal undid the Resume", i)
+ }
+ }
}
// A Resume from another process between an empty pop and its read of the
@@ -2243,18 +2262,22 @@ func TestTotalsSeededFromEndedRows(t *testing.T) {
// A finish that ends no row (repeat, stale or unknown ack) counts nothing.
func TestTotalsIgnoreNoopFinish(t *testing.T) {
- q, qs := openQueue(t)
- defer qs.Close()
- q.Enqueue(job("a"))
- _, ack := pop(t, q)
- if !q.Acknowledge(ack) {
- t.Fatal("Acknowledge failed")
- }
- if q.Acknowledge(ack) || q.Reject(ack, "x") || q.AcknowledgeCancelled(ack, "x") ||
- q.Acknowledge("garbage") || q.Reject("999", "x") {
- t.Error("Expected a finish of no processing row to return false")
- }
- if got, want := stats(t, q).Totals, (contract.JobTotals{Submitted: 1, Completed: 1}); got != want {
- t.Errorf("got %+v, want %+v", got, want)
+ for _, keep := range []bool{false, true} {
+ t.Run(fmt.Sprintf("keep=%t", keep), func(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(!keep))
+ defer qs.Close()
+ q.Enqueue(job("a"))
+ _, ack := pop(t, q)
+ if !q.Acknowledge(ack) {
+ t.Fatal("Acknowledge failed")
+ }
+ if q.Acknowledge(ack) || q.Reject(ack, "x") || q.AcknowledgeCancelled(ack, "x") ||
+ q.Acknowledge("garbage") || q.Reject("999", "x") {
+ t.Error("Expected a finish of no processing row to return false")
+ }
+ if got, want := stats(t, q).Totals, (contract.JobTotals{Submitted: 1, Completed: 1}); got != want {
+ t.Errorf("got %+v, want %+v", got, want)
+ }
+ })
}
}
From 584a58fdcfe5bdcda5f97a7c8d2030cd5fa8e6f6 Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 06:02:17 +0600
Subject: [PATCH 21/22] refactor: setPaused takes only the gen; own calls pass
gen.Add(1)
---
pause.go | 22 ++++++++++------------
queue.go | 2 +-
2 files changed, 11 insertions(+), 13 deletions(-)
diff --git a/pause.go b/pause.go
index b192d34..05388a2 100644
--- a/pause.go
+++ b/pause.go
@@ -23,7 +23,7 @@ import (
// a newer Pause or Resume of this object.
type pauseState struct {
paused atomic.Bool // written under mu
- gen atomic.Uint64 // moves at each Pause and Resume; under mu
+ gen atomic.Uint64 // moves at each Pause and Resume, after its write
refusals atomic.Uint64 // pops refused for a pause
mu sync.Mutex
hooks map[*func(bool)]struct{}
@@ -53,7 +53,7 @@ func (q *Queue) setPauseRow(p bool, query string, args ...any) error {
if err != nil {
return err
}
- q.setPaused(p, true, 0)
+ q.setPaused(p, q.pz.gen.Add(1))
return nil
}
@@ -91,15 +91,13 @@ func (q *Queue) OnPauseChange(fn func(paused bool)) (cancel func()) {
}
}
-// setPaused sets the cache to p: own for this object's Pause or Resume,
-// which moves gen; else only if gen is still g, the gen of the call that
-// read p (a Pause or Resume since may be newer than p). On a change, after
-// mu, it starts the poll (paused) and calls the hooks.
-func (q *Queue) setPaused(p, own bool, g uint64) {
+// setPaused sets the cache to p if gen is still g: the gen of the call
+// that read p (a pop or the poll; a Pause or Resume since may be newer than
+// p), or the one this object's own Pause or Resume moved it to. On a
+// change, after mu, it starts the poll (paused) and calls the hooks.
+func (q *Queue) setPaused(p bool, g uint64) {
q.pz.mu.Lock()
- if own {
- q.pz.gen.Add(1)
- } else if q.pz.gen.Load() != g {
+ if q.pz.gen.Load() != g {
q.pz.mu.Unlock()
return
}
@@ -129,14 +127,14 @@ func (q *Queue) setPaused(p, own bool, g uint64) {
// refused records a pop refused for a pause, by a call that began at gen g.
func (q *Queue) refused(g uint64) {
q.pz.refusals.Add(1)
- q.setPaused(true, false, g)
+ q.setPaused(true, g)
}
// pollPause reads the pause row for the poll.
func (q *Queue) pollPause() {
g := q.pz.gen.Load()
if paused, err := readPaused(q.pausedAt); err == nil {
- q.setPaused(paused, false, g)
+ q.setPaused(paused, g)
}
}
diff --git a/queue.go b/queue.go
index c3d820a..1ae55ed 100644
--- a/queue.go
+++ b/queue.go
@@ -93,7 +93,7 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro
q.closeStmts()
return nil, fmt.Errorf("failed to read pause: %w", err)
} else if paused {
- q.setPaused(true, true, 0)
+ q.setPaused(true, q.pz.gen.Load())
}
return q, nil
From 2bf62726f502497a516a4947c220c3a95c3420fb Mon Sep 17 00:00:00 2001
From: fahimfaisaal
Date: Mon, 9 Nov 2026 06:02:17 +0600
Subject: [PATCH 22/22] docs: recovery runs on every process's first open of a
queue
---
README.md | 2 +-
registry.go | 4 ++++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 09e9ab7..ca8f296 100644
--- a/README.md
+++ b/README.md
@@ -242,7 +242,7 @@ Names are case-insensitive in both layouts: `Jobs` and `jobs` are the same queue
sqliteq keeps a registry in the process, keyed by the file's real path (made absolute, with symlinks resolved). Every `New` in the process that reaches the same file shares one connection pool and one write lock, so two `New` calls on one path never write through two competing pools. Two `Queues` values can therefore hold the same queue, and you get the same handle from both. The file closes when the last `Queues` using it is closed.
-Other processes can open the same file. SQLite's lock orders their writes, and what lives in the file reaches every process: a pause stops every process's pops at once, and a `Resume`, a cancel or a job outcome reaches the others within 50 ms (they poll). Recovery is the limit: the first time a process opens a queue, it puts every item still marked as processing back to pending, so a process that opens a queue while another one runs its jobs reruns those jobs. sqliteq does not take an OS lock to stop that. Open the queue in every process before any of them takes jobs, or let only one process consume it; producers, observers and `Pause`/`Resume` callers do not take jobs.
+Other processes can open the same file. SQLite's lock orders their writes, and what lives in the file reaches every process: a pause stops every process's pops at once, and a `Resume`, a cancel or a job outcome reaches the others within 50 ms (they poll). Recovery is the limit. Every process that opens a queue runs recovery on its first open of that queue, whatever it does with it: consumers, producers, observers (a process that only serves the REST API) and `Pause`/`Resume` callers alike. Recovery puts every item still marked as processing back to pending, so a process that opens the queue while another one runs its jobs makes those jobs run again. sqliteq does not take an OS lock to stop that. Sharing a file is safe only when every process opens the queue before any of them takes jobs, and no process opens it, or opens it again after a restart, later.
On macOS, spell a path the same way every time: the registry key keeps the letter case you pass, so `Data` and `data` would be two entries for one folder. On Windows, Go resolves existing folders and files to their on-disk letter case, so different spellings of them share one entry.
diff --git a/registry.go b/registry.go
index 2d77f2a..c04e8d1 100644
--- a/registry.go
+++ b/registry.go
@@ -184,6 +184,10 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error)
h.q = q
}
+ // ponytail: recovery runs on every process's first open, also one that
+ // never takes jobs (a producer, observer or pauser), and requeues jobs
+ // another process is running. Upgrade path: owner ids plus heartbeats
+ // on processing rows, requeueing only rows of dead owners (redisq's model).
if !f.recovered[key] {
if err := h.queue().requeueNoAck(); err != nil {
h.queue().Close() // stops a pause poll