From 35cf8ade05fcef9fd437d042bb6481d810c4c613 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 21:30:05 +0600 Subject: [PATCH 01/10] build: pin varmq and contract at 0a9288e Go 1.27 (varmq root needs it). contract pinned too: varmq's replace is not inherited. --- go.mod | 8 ++++++-- go.sum | 12 ++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b6ff707..6b04aeb 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,9 @@ module github.com/goptics/sqliteq -go 1.24 +go 1.27 -require github.com/mattn/go-sqlite3 v1.14.28 +require ( + github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092 + github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092 + github.com/mattn/go-sqlite3 v1.14.28 +) diff --git a/go.sum b/go.sum index 42e5bac..f4edda0 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,14 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092 h1:oFPxzb1mXotIzHHKlxz+kUGO1ASrWZkg2nmkYhrgx60= +github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E= +github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092 h1:wWq1t3n0//1/URO+uwwdEybf6GJcVHmmtJ9lcKAHxRY= +github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 010dd7c472ad48b21c1e8490954648aeffdbf370 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 21:30:05 +0600 Subject: [PATCH 02/10] feat: job outcome and cancel for varmq OutcomeWatcher, ResultAcknowledgeable, JobCanceller, CancelNotifier, CancelAcknowledgeable on Queue and PriorityQueue (spec 6.13, 6.14). Sidecar tables __sqliteq_jobs (outcome per Job ID, retention) and __sqliteq_cancelled (marks, AUTOINCREMENT seq). Job ID = job_id, else item JSON id. Finish writes outcome + clears mark in its tx. Pops gate on live marks, drop with outcome cancelled, cap 1000. EnqueueOnce re-add clears both via AFTER INSERT trigger. Purge clears both. No pub/sub: one poll goroutine per queue object, 50 ms, only while watches or hooks exist. Txs begin IMMEDIATE. --- cancel.go | 172 ++++++++++++++++ outcome.go | 326 ++++++++++++++++++++++++++++++ priority_queue_test.go | 44 ++++ queue.go | 245 ++++++++++++++++------ queue_test.go | 446 +++++++++++++++++++++++++++++++++++++++++ registry.go | 9 +- 6 files changed, 1181 insertions(+), 61 deletions(-) create mode 100644 cancel.go create mode 100644 outcome.go diff --git a/cancel.go b/cancel.go new file mode 100644 index 0000000..ce8f1c9 --- /dev/null +++ b/cancel.go @@ -0,0 +1,172 @@ +package sqliteq + +import ( + "database/sql" + "errors" + "fmt" + "time" + + "github.com/goptics/varmq/contract" +) + +// Job cancel (varmq spec/redis-storage.md 1.7.0, 6.14), on SQLite. +// CancelJob writes a mark to __sqliteq_cancelled (expires_at: ms). +// Every pop drops a marked waiting job with outcome cancelled; OnCancel +// learns of marks by polling seq; AcknowledgeCancelled finishes a running +// job that stopped for it. Any finish, and an EnqueueOnce that stores the +// id, clears the mark. sqliteq has no workflow store, so no item is exempt. + +var ( + _ contract.JobCanceller = (*Queue)(nil) + _ contract.JobCanceller = (*PriorityQueue)(nil) + _ contract.CancelNotifier = (*Queue)(nil) + _ contract.CancelNotifier = (*PriorityQueue)(nil) + _ contract.CancelAcknowledgeable = (*Queue)(nil) + _ contract.CancelAcknowledgeable = (*PriorityQueue)(nil) +) + +const ( + // cancelTTL is how long one mark lasts. + cancelTTL = 24 * time.Hour + // maxDrops caps the marked items one pop drops; at the cap it returns + // what it has and the next pop goes on. + maxDrops = 1000 +) + +var ( + errEmptyJobID = errors.New("sqliteq: empty job id") + errClosed = errors.New("sqliteq: queue is closed") +) + +// CancelJob implements contract.JobCanceller: it marks job id cancelled for +// cancelTTL. A waiting job is dropped by the pop that would hand it out +// (outcome cancelled); OnCancel callbacks of every queue object on the file +// learn of a running one, which ends only if its worker stops it. +// contract.ErrJobEnded when its outcome record shows it ended. An unknown +// id is marked too: a later Enqueue of it is dropped, unless EnqueueOnce +// stores it (that clears the mark). +func (q *Queue) CancelJob(id string) error { + if id == "" { + return errEmptyJobID + } + if q.closed.Load() { + return errClosed + } + + now := time.Now().UnixMilli() + q.f.mu.Lock() + defer q.f.mu.Unlock() + tx, err := q.f.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + var state string + switch err := tx.QueryRow(fmt.Sprintf("SELECT state FROM %s WHERE job_id = ?", q.jobs), id).Scan(&state); { + case err == nil: + return contract.ErrJobEnded // every record is an end + case !errors.Is(err, sql.ErrNoRows): + return err + } + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE expires_at <= ?", q.cancelled), now); err != nil { + return err + } + // REPLACE deletes an older mark of id, so the new one takes a new seq + // and reaches OnCancel again + if _, err := tx.Exec(fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, expires_at) VALUES (?, ?)", q.cancelled), + id, now+cancelTTL.Milliseconds()); err != nil { + return err + } + return tx.Commit() +} + +// AcknowledgeCancelled implements contract.CancelAcknowledgeable: Reject +// with outcome cancelled and reason as its error. A kept row +// (WithRemoveOnComplete(false)) gets status 'cancelled'. +func (q *Queue) AcknowledgeCancelled(ackID, reason string) bool { + return q.finish(ackID, contract.JobCancelled, reason, nil) +} + +// OnCancel implements contract.CancelNotifier: fn gets the job ID of each +// cancel request made after the call, through any queue object on this +// database file, until stop. Marks are polled every pollEvery; each call +// runs on its own goroutine. A mark gone by the poll (the job finished or +// was re-added) is never reported. A closed queue reports nothing. +func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) { + // Read before the lock: a mark made after this read has a higher seq + var last sql.NullInt64 + _ = q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.cancelled)).Scan(&last) + + p := &fn + w := &q.w + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return func() {} + } + if len(w.hooks) == 0 { + w.lastSeq = last.Int64 + } + if w.hooks == nil { + w.hooks = map[*func(string)]struct{}{} + } + w.hooks[p] = struct{}{} + w.startLocked(q) + + return func() { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.hooks, p) + } +} + +// pollCancels passes the live marks newer than seq to hooks. +func (q *Queue) pollCancels(seq int64, hooks []func(string)) { + rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? AND expires_at > ? ORDER BY seq LIMIT %d", + q.cancelled, maxDrops), seq, time.Now().UnixMilli()) + if err != nil { + return + } + var ids []string + for rows.Next() { + var id string + if rows.Scan(&seq, &id) != nil { + continue + } + ids = append(ids, id) + } + if rows.Close() != nil || rows.Err() != nil { + return // read again from the old seq + } + + w := &q.w + w.mu.Lock() + w.lastSeq = max(w.lastSeq, seq) + w.mu.Unlock() + for _, id := range ids { + for _, fn := range hooks { + go fn(id) + } + } +} + +// anyMark reports whether a live mark exists, so a pop without marks pays +// one read; an error counts as a mark (the gated pop then reports it). +func (q *Queue) anyMark(now time.Time) bool { + var n int + if err := q.hasMark.QueryRow(now.UnixMilli()).Scan(&n); err != nil { + return true + } + return n == 1 +} + +// marked reports whether job id has a live mark, in tx. +func (q *Queue) marked(tx *sql.Tx, id string, now time.Time) (bool, error) { + if id == "" { + return false, nil + } + var n int + err := tx.Stmt(q.isMarked).QueryRow(id, now.UnixMilli()).Scan(&n) + return n == 1, err +} diff --git a/outcome.go b/outcome.go new file mode 100644 index 0000000..5aa91e4 --- /dev/null +++ b/outcome.go @@ -0,0 +1,326 @@ +package sqliteq + +import ( + "database/sql" + "encoding/json" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/goptics/varmq/contract" +) + +// Job outcome (varmq spec/redis-storage.md 1.6.0, 6.13), on SQLite. Each +// finish of a job with a Job ID writes one row to __sqliteq_jobs in +// the finish's transaction. SQLite has no pub/sub, so a watcher is +// answered by polling that table: one goroutine per queue object, alive +// only while it has watches or OnCancel hooks. + +var ( + _ contract.OutcomeWatcher = (*Queue)(nil) + _ contract.OutcomeWatcher = (*PriorityQueue)(nil) + _ contract.ResultAcknowledgeable = (*Queue)(nil) + _ contract.ResultAcknowledgeable = (*PriorityQueue)(nil) +) + +const ( + // pollEvery is how often a queue object with watches or hooks reads + // the outcome and cancel tables. + pollEvery = 50 * time.Millisecond + // maxCatchUp caps the watched ids one poll reads; map order rotates a + // larger set. + maxCatchUp = 500 + + // Retention (spec 5 defaults): cancelled records count as failed. + keepCompleted = 1000 + keepCompletedAge = 24 * time.Hour + keepFailed = 10000 + keepFailedAge = 7 * 24 * time.Hour + // trimEvery is how many records a queue object writes between trims. + trimEvery = 100 + + jobsSuffix = "__sqliteq_jobs" + cancelledSuffix = "__sqliteq_cancelled" + readdSuffix = "__sqliteq_readd" +) + +// initSidecars creates the outcome and cancel tables, and the trigger that +// clears both for an id EnqueueOnce stores (spec 6.13 and 6.14 Re-add): an +// ON CONFLICT DO NOTHING insert of a repeat id fires no trigger. The caller +// holds the write mutex. +func (q *Queue) initSidecars() error { + for _, s := range []string{ + `CREATE TABLE IF NOT EXISTS %[2]s ( + job_id TEXT PRIMARY KEY, + state TEXT NOT NULL, + error TEXT, + result BLOB, + finished_at INTEGER NOT NULL + ) WITHOUT ROWID`, + `CREATE INDEX IF NOT EXISTS %[4]s ON %[2]s (finished_at)`, + // AUTOINCREMENT: seq never reuses a deleted mark's, so OnCancel's + // seq > last cursor sees every new mark + `CREATE TABLE IF NOT EXISTS %[3]s ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL UNIQUE, + expires_at INTEGER NOT NULL + )`, + `CREATE TRIGGER IF NOT EXISTS %[5]s AFTER INSERT ON %[1]s WHEN NEW.job_id IS NOT NULL BEGIN + DELETE FROM %[2]s WHERE job_id = NEW.job_id; + DELETE FROM %[3]s WHERE job_id = NEW.job_id; + END`, + } { + query := fmt.Sprintf(s, q.table, q.jobs, q.cancelled, + quoteIdent(q.name+jobsSuffix+"_finished"), quoteIdent(q.name+readdSuffix)) + if _, err := q.f.db.Exec(query); err != nil { + return err + } + } + return nil +} + +// jobIDOf is a row's Job ID: its EnqueueOnce id, else the envelope "id" of +// a JSON object item (varmq's generated ids come with plain Enqueue). "" +// when it has neither: such a job has no outcome and is never cancelled. +func jobIDOf(onceID sql.NullString, data []byte) string { + if onceID.Valid { + return onceID.String + } + if len(data) == 0 || data[0] != '{' { + return "" + } + var env struct { + ID string `json:"id"` + } + if json.Unmarshal(data, &env) != nil { + return "" + } + return env.ID +} + +// record writes job id's outcome and drops its cancel mark, in tx; every +// trimEvery-th record also trims both states. +func (q *Queue) record(tx *sql.Tx, id string, state contract.JobState, reason string, result []byte, now time.Time) error { + var errText any + if state != contract.JobCompleted { + errText = reason + } + if _, err := tx.Stmt(q.putJob).Exec(id, string(state), errText, result, now.UnixMilli()); err != nil { + return err + } + if _, err := tx.Stmt(q.unmark).Exec(id); err != nil { + return err + } + if q.records.Add(1)%trimEvery == 0 { + return q.trim(tx, now) + } + return nil +} + +// trim applies retention: per state group, records past the age limit and +// those past the newest N. +func (q *Queue) trim(tx *sql.Tx, now time.Time) error { + for _, g := range []struct { + where string + keep int + age time.Duration + }{ + {"state = 'completed'", keepCompleted, keepCompletedAge}, + {"state <> 'completed'", keepFailed, keepFailedAge}, + } { + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE %s AND finished_at <= ?", q.jobs, g.where), + now.Add(-g.age).UnixMilli()); err != nil { + return err + } + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %[1]s WHERE job_id IN (SELECT job_id FROM %[1]s WHERE %[2]s ORDER BY finished_at DESC LIMIT -1 OFFSET %[3]d)", + q.jobs, g.where, g.keep)); err != nil { + return err + } + } + return nil +} + +// AcknowledgeResult implements contract.ResultAcknowledgeable: Acknowledge +// that keeps result, which must be valid JSON, in the job's outcome. Not +// JSON: false, and nothing changes. +func (q *Queue) AcknowledgeResult(ackID string, result []byte) bool { + if !json.Valid(result) { + return false + } + return q.finish(ackID, contract.JobCompleted, "", result) +} + +type outcomeWatch struct { + fn func(contract.JobOutcome) + done atomic.Bool // called or cancelled +} + +// watchers are a queue object's outcome watches and OnCancel hooks, and the +// state of its one poll goroutine. mu is never held across a query. +type watchers struct { + mu sync.Mutex + byID map[string]map[*outcomeWatch]struct{} + hooks map[*func(string)]struct{} + lastSeq int64 // newest cancel mark seq OnCancel has seen + polling bool + closed bool + stop chan struct{} // closed by close +} + +// startLocked starts the poll goroutine if none runs; w.mu is held. +func (w *watchers) startLocked(q *Queue) { + if !w.polling { + w.polling = true + go q.poll() + } +} + +// close drops every watch and hook without calling it, and stops polling. +func (w *watchers) close() { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + w.closed = true + close(w.stop) + for _, set := range w.byID { + for o := range set { + o.done.Store(true) + } + } + w.byID, w.hooks = nil, nil +} + +// WatchOutcome implements contract.OutcomeWatcher: fn runs once, on its own +// goroutine, when job id completes, fails or is cancelled through any +// queue object on this database file, in this process or another. A job +// whose record is still kept is reported even if it ended before the call. +// It is polled every pollEvery. An empty id, or a closed queue, watches +// nothing. Close drops watches without calling fn. +func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel func()) { + o := &outcomeWatch{fn: fn} + w := &q.w + w.mu.Lock() + if id == "" || w.closed { + w.mu.Unlock() + return func() {} + } + if w.byID == nil { + w.byID = map[string]map[*outcomeWatch]struct{}{} + } + if w.byID[id] == nil { + w.byID[id] = map[*outcomeWatch]struct{}{} + } + w.byID[id][o] = struct{}{} + w.startLocked(q) + w.mu.Unlock() + + go q.catchUp([]string{id}) + + return func() { + if o.done.CompareAndSwap(false, true) { + w.mu.Lock() + defer w.mu.Unlock() + if set := w.byID[id]; set != nil { + delete(set, o) + if len(set) == 0 { + delete(w.byID, id) + } + } + } + } +} + +// poll reads the watched outcomes and new cancel marks every pollEvery, +// until Close or until nothing is watched. +func (q *Queue) poll() { + t := time.NewTicker(pollEvery) + defer t.Stop() + w := &q.w + for { + select { + case <-w.stop: + return + case <-t.C: + } + + w.mu.Lock() + if len(w.byID) == 0 && len(w.hooks) == 0 { + w.polling = false + w.mu.Unlock() + return + } + ids := make([]string, 0, min(len(w.byID), maxCatchUp)) + for id := range w.byID { + if len(ids) == maxCatchUp { + break + } + ids = append(ids, id) + } + hooks := make([]func(string), 0, len(w.hooks)) + for fn := range w.hooks { + hooks = append(hooks, *fn) + } + seq := w.lastSeq + w.mu.Unlock() + + if len(ids) > 0 { + q.catchUp(ids) + } + if len(hooks) > 0 { + q.pollCancels(seq, hooks) + } + } +} + +// catchUp delivers the ids whose outcome record exists. A failed read is +// retried by the next poll. +func (q *Queue) catchUp(ids []string) { + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + rows, err := q.f.db.Query(fmt.Sprintf("SELECT job_id, state, error, result FROM %s WHERE job_id IN (%s)", + q.jobs, strings.Repeat("?, ", len(ids)-1)+"?"), args...) + if err != nil { + return + } + var outs []contract.JobOutcome + for rows.Next() { + var ( + id, state string + errText sql.NullString + result []byte + ) + if rows.Scan(&id, &state, &errText, &result) != nil { + continue + } + outs = append(outs, contract.JobOutcome{ID: id, State: contract.JobState(state), Error: errText.String, Result: result}) + } + rows.Close() + + for _, out := range outs { + q.deliver(out) + } +} + +// deliver ends every watch of out.ID: each fn runs once, on its own +// goroutine, so a slow fn holds up no poll. +func (q *Queue) deliver(out contract.JobOutcome) { + w := &q.w + w.mu.Lock() + set := w.byID[out.ID] + delete(w.byID, out.ID) + w.mu.Unlock() + + for o := range set { + go func() { + if o.done.CompareAndSwap(false, true) { + o.fn(out) + } + }() + } +} diff --git a/priority_queue_test.go b/priority_queue_test.go index 39b2ed7..a7cadcf 100644 --- a/priority_queue_test.go +++ b/priority_queue_test.go @@ -1,10 +1,13 @@ package sqliteq import ( + "errors" "fmt" "strconv" "testing" + "github.com/goptics/varmq/contract" + _ "github.com/mattn/go-sqlite3" ) @@ -653,3 +656,44 @@ func TestBatchPriorityOrder(t *testing.T) { } } } + +// A priority queue reports outcomes, drops marked waiting jobs in priority +// order, and re-add clears the mark, as Queue does. +func TestPriorityOutcomeAndCancel(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("pq") + if err != nil { + t.Fatal(err) + } + other := otherProcess(t, pq.Queue) + + pq.Enqueue(job("low"), 5) + pq.Enqueue(job("high"), 1) + pq.EnqueueOnce(job("mid"), 3, "mid") + if err := other.CancelJob("high"); err != nil { + t.Fatal(err) + } + chHigh, chMid := watch(other, "high"), watch(other, "mid") + + if v, ack := pop(t, pq.Queue); v != string(job("mid")) { + t.Fatalf("got %s, want mid", v) + } else if !pq.AcknowledgeResult(ack, []byte(`7`)) { + t.Fatal("AcknowledgeResult failed") + } + if got := waitOutcome(t, chHigh); got.State != contract.JobCancelled { + t.Errorf("high: got %+v", got) + } + if got := waitOutcome(t, chMid); got.State != contract.JobCompleted || string(got.Result) != "7" { + t.Errorf("mid: got %+v", got) + } + if err := pq.CancelJob("mid"); !errors.Is(err, contract.ErrJobEnded) { + t.Errorf("got %v, want ErrJobEnded", err) + } + + pq.CancelJob("again") + pq.EnqueueOnce(job("again"), 0, "again") + if v, _ := pop(t, pq.Queue); v != string(job("again")) { + t.Errorf("Expected the re-added job first, got %s", v) + } +} diff --git a/queue.go b/queue.go index 7d41be6..471326f 100644 --- a/queue.go +++ b/queue.go @@ -4,6 +4,7 @@ import ( "cmp" "database/sql" "database/sql/driver" + "errors" "fmt" "slices" "strconv" @@ -11,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/goptics/varmq/contract" _ "github.com/mattn/go-sqlite3" ) @@ -26,8 +28,13 @@ type Queue struct { removeOnComplete bool closed atomic.Bool + jobs, cancelled string // quoted outcome and cancel mark tables + w watchers // outcome watches and OnCancel hooks + records atomic.Int64 // outcome records written, for trims + // Prepared once per handle; they close with the file's pool - insert, claim, pop, ack, reject, count, countCapped, values *sql.Stmt + insert, claim, pop, fin, count, countCapped, values *sql.Stmt + putJob, unmark, hasMark, isMarked *sql.Stmt } // claimed is one row a claim moved to processing @@ -35,6 +42,7 @@ type claimed struct { id int64 priority int data []byte + once sql.NullString // job_id } // newQueue creates a SQLite-based queue on its table: the table itself, the @@ -48,7 +56,10 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro priority: priority, order: "id", removeOnComplete: true, // Default to removing completed items + jobs: quoteIdent(name + jobsSuffix), + cancelled: quoteIdent(name + cancelledSuffix), } + q.w.stop = make(chan struct{}) if priority { q.order = "priority, id" } @@ -86,6 +97,9 @@ func (q *Queue) initSchema() error { if err := q.addJobID(); err != nil { return fmt.Errorf("failed to add job_id: %w", err) } + if err := q.initSidecars(); err != nil { + return fmt.Errorf("failed to create outcome tables: %w", err) + } return nil } @@ -198,18 +212,16 @@ func (q *Queue) prepare() (err error) { } }() - cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data" + cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data, job_id" if q.priority { - cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data" + cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data, job_id" } pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order) - ack := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing'", q.table) - reject := ack + // fin ends a processing row: deleted, or kept with its end as status + fin := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table) if !q.removeOnComplete { - finish := "UPDATE %s SET status = '%s', ack = 1, updated_at = ? WHERE id = ? AND status = 'processing'" - ack = fmt.Sprintf(finish, q.table, "completed") - reject = fmt.Sprintf(finish, q.table, "failed") + fin = fmt.Sprintf("UPDATE %s SET status = ?, ack = 1, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table) } for _, s := range []struct { @@ -220,9 +232,12 @@ func (q *Queue) prepare() (err error) { // other failure still errors; it must repeat the index's WHERE {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING", q.table, cols, vals)}, {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, - {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)}, - {&q.ack, ack}, - {&q.reject, reject}, + {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data, job_id", q.table, pending)}, + {&q.fin, fin}, + {&q.putJob, fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, state, error, result, finished_at) VALUES (?, ?, ?, ?, ?)", q.jobs)}, + {&q.unmark, fmt.Sprintf("DELETE FROM %s WHERE job_id = ?", q.cancelled)}, + {&q.hasMark, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE expires_at > ?)", q.cancelled)}, + {&q.isMarked, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE job_id = ? AND expires_at > ?)", q.cancelled)}, {&q.count, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", q.table)}, {&q.countCapped, fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE status = 'pending' LIMIT %d)", q.table, remainingCap)}, {&q.values, fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)}, @@ -237,7 +252,7 @@ func (q *Queue) prepare() (err error) { // closeStmts closes the statements prepared so far func (q *Queue) closeStmts() { - for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.ack, q.reject, q.count, q.countCapped, q.values} { + for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.fin, q.count, q.countCapped, q.values, q.putJob, q.unmark, q.hasMark, q.isMarked} { if s != nil { s.Close() } @@ -351,7 +366,7 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { // returns them in queue order func (q *Queue) claimN(n int) ([]claimed, error) { q.f.mu.Lock() - out, err := q.claimRows(n) + out, err := q.claimGated(n) q.f.mu.Unlock() if err != nil { return nil, err @@ -367,10 +382,56 @@ func (q *Queue) claimN(n int) ([]claimed, error) { return out, nil } -// claimRows runs the claim statement and reads every row it returns; the +// claimGated claims up to n rows; with a live cancel mark on the file it +// runs in one transaction that drops each marked row it claims (outcome +// cancelled) and claims again, at most maxDrops drops. The caller holds the +// write mutex. +func (q *Queue) claimGated(n int) ([]claimed, error) { + now := time.Now() + if !q.anyMark(now) { + return q.claimRows(q.claim, n) + } + + tx, err := q.f.db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + claim := tx.Stmt(q.claim) + out := make([]claimed, 0, n) + for drops := 0; len(out) < n && drops < maxDrops; { + rows, err := q.claimRows(claim, n-len(out)) + if err != nil || len(rows) == 0 { + if err != nil { + return nil, err + } + break + } + // Every claimed row is handed out or dropped, so a batch may pass the cap + for _, c := range rows { + id := jobIDOf(c.once, c.data) + drop, err := q.marked(tx, id, now) + if err != nil { + return nil, err + } + if !drop { + out = append(out, c) + continue + } + if _, err := q.end(tx, c.id, id, contract.JobCancelled, "cancelled", nil, now); err != nil { + return nil, err + } + drops++ + } + } + return out, tx.Commit() +} + +// claimRows runs a claim statement and reads every row it returns; the // caller holds the write mutex -func (q *Queue) claimRows(n int) ([]claimed, error) { - rows, err := q.claim.Query(time.Now().UTC(), n) +func (q *Queue) claimRows(claim *sql.Stmt, n int) ([]claimed, error) { + rows, err := claim.Query(time.Now().UTC(), n) if err != nil { return nil, err } @@ -380,9 +441,9 @@ func (q *Queue) claimRows(n int) ([]claimed, error) { for rows.Next() { var c claimed if q.priority { - err = rows.Scan(&c.id, &c.priority, &c.data) + err = rows.Scan(&c.id, &c.priority, &c.data, &c.once) } else { - err = rows.Scan(&c.id, &c.data) + err = rows.Scan(&c.id, &c.data, &c.once) } if err != nil { return nil, err @@ -403,25 +464,45 @@ func (q *Queue) Dequeue() (any, bool) { q.f.mu.Lock() defer q.f.mu.Unlock() - // Deferred last, so the rows close (and the statement ends) before unlocking - rows, err := q.pop.Query() + now := time.Now() + if !q.anyMark(now) { + data, _, err := popRow(q.pop) + return data, err == nil + } + + // Gated: a marked item is dropped (outcome cancelled) and the next one popped + tx, err := q.f.db.Begin() if err != nil { return nil, false } - defer rows.Close() - - var data []byte - found := false - for rows.Next() { - if err := rows.Scan(&data); err != nil { + defer tx.Rollback() + pop := tx.Stmt(q.pop) + for range maxDrops { + data, once, err := popRow(pop) + if err != nil { + break + } + id := jobIDOf(once, data) + drop, err := q.marked(tx, id, now) + if err != nil { + return nil, false + } + if !drop { + return data, tx.Commit() == nil + } + if err := q.record(tx, id, contract.JobCancelled, "cancelled", nil, now); err != nil { return nil, false } - found = true - } - if rows.Err() != nil || !found { - return nil, false } - return data, true + tx.Commit() // keep the drops; an empty queue or the cap hands out nothing + return nil, false +} + +// popRow runs a pop statement: the deleted row's data and job_id, or +// sql.ErrNoRows on an empty queue +func popRow(pop *sql.Stmt) (data []byte, once sql.NullString, err error) { + err = pop.QueryRow().Scan(&data, &once) + return data, once, err } // DequeueWithAckId moves the next item to processing and returns it with its @@ -471,46 +552,77 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini // Acknowledge marks a processing item as completed, deleting it unless // WithRemoveOnComplete(false) is set, in the file's next group commit round // Returns true only if exactly one processing row had this ack ID -func (q *Queue) Acknowledge(ackID string) bool { return q.finish(ackID, q.ack) } +func (q *Queue) Acknowledge(ackID string) bool { + return q.finish(ackID, contract.JobCompleted, "", nil) +} // Reject marks a processing item as failed for good: it is not requeued. // The row is deleted, or with WithRemoveOnComplete(false) kept with status -// 'failed'. Same group commit and result as Acknowledge. -// ponytail: reason is not stored (no column); add one when something reads it. -func (q *Queue) Reject(ackID string, reason string) bool { return q.finish(ackID, q.reject) } +// 'failed'. reason is the error of the job's outcome. Same group commit and +// result as Acknowledge. +func (q *Queue) Reject(ackID string, reason string) bool { + return q.finish(ackID, contract.JobFailed, reason, nil) +} -// finish runs stmt (ack or reject) for the row of ackID. -func (q *Queue) finish(ackID string, stmt *sql.Stmt) bool { +// finish ends the processing row of ackID with state in the file's next +// group commit round. A round of one runs in its own transaction: the row +// and its outcome record commit together. +func (q *Queue) finish(ackID string, state contract.JobState, reason string, result []byte) bool { id, ok := parseAckID(ackID) if !ok { return false } - now := time.Now().UTC() - return q.f.b.do(func(tx *sql.Tx) (bool, error) { - var ( - result sql.Result - err error - ) - s := stmt - if tx != nil { - s = tx.Stmt(s) + now := time.Now() + return q.f.b.do(func(tx *sql.Tx) (ok bool, err error) { + if tx == nil { + if tx, err = q.f.db.Begin(); err != nil { + return false, err + } + defer func() { + if err == nil { + err = tx.Commit() + } else { + tx.Rollback() + } + ok = ok && err == nil + }() } - if q.removeOnComplete { - result, err = s.Exec(id) - } else { - result, err = s.Exec(now, id) - } - if err != nil { - return false, err - } - // No row is an outcome, not an error: it must not fail the round - n, err := result.RowsAffected() - return err == nil && n == 1, err + return q.end(tx, id, "", state, reason, result, now) }) } +// end finishes processing row id with state, in tx: the row is deleted or +// kept with state as status, and a job with a Job ID gets its outcome +// record. jobID is the row's, when the caller knows it. Reports whether +// the row was processing. +func (q *Queue) end(tx *sql.Tx, id int64, jobID string, state contract.JobState, reason string, result []byte, now time.Time) (bool, error) { + var row *sql.Row + if q.removeOnComplete { + row = tx.Stmt(q.fin).QueryRow(id) + } else { + row = tx.Stmt(q.fin).QueryRow(string(state), now.UTC(), id) + } + var ( + once sql.NullString + data []byte + ) + switch err := row.Scan(&once, &data); { + case errors.Is(err, sql.ErrNoRows): + return false, nil + case err != nil: + return false, err + } + if jobID == "" { + jobID = jobIDOf(once, data) + } + if jobID == "" { + return true, nil + } + return true, q.record(tx, jobID, state, reason, result, now) +} + // Len returns the number of pending items in the queue func (q *Queue) Len() int { var count int @@ -543,17 +655,30 @@ func (q *Queue) Values() []any { return items } -// Purge removes all items from the queue +// Purge removes all items from the queue, with its outcome records and +// cancel marks (spec 6.6, 6.14) func (q *Queue) Purge() { q.f.mu.Lock() defer q.f.mu.Unlock() - q.f.db.Exec(fmt.Sprintf("DELETE FROM %s", q.table)) + tx, err := q.f.db.Begin() + if err != nil { + return + } + defer tx.Rollback() + for _, t := range []string{q.table, q.jobs, q.cancelled} { + if _, err := tx.Exec("DELETE FROM " + t); err != nil { + return + } + } + tx.Commit() } -// Close closes the queue and its database connection +// Close closes the queue and drops its outcome watches and OnCancel hooks +// without calling them. The database file stays open for the Queues. func (q *Queue) Close() error { q.closed.Store(true) + q.w.close() return nil } diff --git a/queue_test.go b/queue_test.go index bf66ab0..68775ff 100644 --- a/queue_test.go +++ b/queue_test.go @@ -1,7 +1,9 @@ package sqliteq import ( + "context" "database/sql" + "errors" "fmt" "path/filepath" "strconv" @@ -9,6 +11,9 @@ import ( "sync/atomic" "testing" "time" + + "github.com/goptics/varmq" + "github.com/goptics/varmq/contract" ) func TestSQLiteQueue(t *testing.T) { @@ -844,6 +849,9 @@ func TestWritesTakeFileMutex(t *testing.T) { {"DequeueWithAckId", func() { q.DequeueWithAckId() }}, {"DequeueBatchWithAckIds", func() { q.DequeueBatchWithAckIds(make([]any, 2), make([]string, 2)) }}, {"Acknowledge", func() { q.Acknowledge(ack1) }}, + {"CancelJob", func() { q.CancelJob("x") }}, + {"gated DequeueWithAckId", func() { q.DequeueWithAckId() }}, + {"gated Dequeue", func() { q.Dequeue() }}, {"requeueNoAck", func() { q.requeueNoAck() }}, {"Purge", func() { q.Purge() }}, } @@ -894,3 +902,441 @@ func TestWritesTakeFileMutex(t *testing.T) { }) } } + +// otherProcess opens q's table again on a pool of its own, as a second +// process on the same database file would. +func otherProcess(t *testing.T, q *Queue) *Queue { + t.Helper() + db, err := sql.Open(driverName(q.f.pages), dsn(q.f.key)) + if err != nil { + t.Fatal(err) + } + f := &file{key: q.f.key, db: db, pages: q.f.pages, refs: 1, handles: map[string]*handle{}, recovered: map[string]bool{}} + f.b = &batcher{commit: f.commitRound} + q2, err := newQueue(f, q.name, q.priority, WithRemoveOnComplete(q.removeOnComplete)) + if err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + q2.Close() + db.Close() + }) + return q2 +} + +// job is a varmq-style envelope with Job ID id. +func job(id string) []byte { return []byte(`{"id":"` + id + `","data":1}`) } + +// watch starts a watch of id on q; the channel gets its one outcome. +func watch(q *Queue, id string) <-chan contract.JobOutcome { + ch := make(chan contract.JobOutcome, 2) + q.WatchOutcome(id, func(o contract.JobOutcome) { ch <- o }) + return ch +} + +func waitOutcome(t *testing.T, ch <-chan contract.JobOutcome) contract.JobOutcome { + t.Helper() + select { + case o := <-ch: + return o + case <-time.After(5 * time.Second): + t.Fatal("no outcome") + return contract.JobOutcome{} + } +} + +func noOutcome(t *testing.T, ch <-chan contract.JobOutcome) { + t.Helper() + select { + case o := <-ch: + t.Fatalf("unexpected outcome %+v", o) + case <-time.After(4 * pollEvery): + } +} + +// pop hands out the next item and its ack id, failing on none. +func pop(t *testing.T, q *Queue) (string, string) { + t.Helper() + v, ok, ack := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId found nothing") + } + return string(v.([]byte)), ack +} + +// A watch reports each way a job ends: Acknowledge, Reject with its reason, +// AcknowledgeResult with its JSON; a job without a Job ID writes nothing. +func TestWatchOutcome(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + cases := []struct { + id string + finish func(ack string) bool + want contract.JobOutcome + }{ + {"ack", q.Acknowledge, contract.JobOutcome{ID: "ack", State: contract.JobCompleted}}, + {"rej", func(a string) bool { return q.Reject(a, "boom") }, contract.JobOutcome{ID: "rej", State: contract.JobFailed, Error: "boom"}}, + {"res", func(a string) bool { return q.AcknowledgeResult(a, []byte(`{"n":42}`)) }, + contract.JobOutcome{ID: "res", State: contract.JobCompleted, Result: []byte(`{"n":42}`)}}, + } + for _, c := range cases { + ch := watch(q, c.id) + q.Enqueue(job(c.id)) + _, ack := pop(t, q) + noOutcome(t, ch) + if !c.finish(ack) { + t.Fatalf("%s: finish failed", c.id) + } + got := waitOutcome(t, ch) + if got.ID != c.want.ID || got.State != c.want.State || got.Error != c.want.Error || string(got.Result) != string(c.want.Result) { + t.Errorf("%s: got %+v, want %+v", c.id, got, c.want) + } + noOutcome(t, ch) // once + } + + q.Enqueue([]byte("plain")) + _, ack := pop(t, q) + q.Acknowledge(ack) + if n := countJobs(t, q); n != 3 { + t.Errorf("Expected 3 outcome records, got %d", n) + } +} + +func countJobs(t *testing.T, q *Queue) int { + t.Helper() + var n int + if err := q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.jobs).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// AcknowledgeResult with invalid JSON fails and changes nothing. +func TestAcknowledgeResultInvalid(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.Enqueue(job("a")) + _, ack := pop(t, q) + if q.AcknowledgeResult(ack, []byte(`{nope`)) { + t.Fatal("Expected invalid JSON to fail") + } + if n := countRows(t, q.f.db, "status = 'processing'"); n != 1 { + t.Errorf("Expected the row still processing, got %d", n) + } + if !q.Acknowledge(ack) { + t.Error("Expected Acknowledge to finish it") + } +} + +// A job that ended before the watch is reported from its record; cancel +// stops a watch; a closed queue watches nothing. +func TestWatchEndedBeforeAndCancel(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.Enqueue(job("early")) + _, ack := pop(t, q) + q.Reject(ack, "late watcher") + if got := waitOutcome(t, watch(q, "early")); got.State != contract.JobFailed || got.Error != "late watcher" { + t.Errorf("got %+v", got) + } + + ch := make(chan contract.JobOutcome, 1) + stop := q.WatchOutcome("never", func(o contract.JobOutcome) { ch <- o }) + stop() + stop() + q.Enqueue(job("never")) + _, ack = pop(t, q) + q.Acknowledge(ack) + noOutcome(t, ch) + + q.Close() + q.WatchOutcome("early", func(o contract.JobOutcome) { ch <- o })() + noOutcome(t, ch) +} + +// Two queue objects on one file, as two processes: each learns of jobs the +// other finishes, and a cancel reaches the other's pops and OnCancel. +func TestOutcomeAcrossProcesses(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q2 := otherProcess(t, q) + + ch := watch(q, "x") + q.Enqueue(job("x")) + _, ack := pop(t, q2) + q2.Acknowledge(ack) + if got := waitOutcome(t, ch); got.ID != "x" || got.State != contract.JobCompleted { + t.Errorf("got %+v", got) + } + + ids := make(chan string, 4) + defer q2.OnCancel(func(id string) { ids <- id })() + q2.Enqueue(job("run")) + _, ack = pop(t, q2) + ch = watch(q2, "run") + if err := q.CancelJob("run"); err != nil { + t.Fatal(err) + } + select { + case id := <-ids: + if id != "run" { + t.Errorf("OnCancel got %q", id) + } + case <-time.After(5 * time.Second): + t.Fatal("OnCancel did not fire") + } + if !q2.AcknowledgeCancelled(ack, "stopped") { + t.Fatal("AcknowledgeCancelled failed") + } + if got := waitOutcome(t, ch); got.State != contract.JobCancelled || got.Error != "stopped" { + t.Errorf("got %+v", got) + } + var marks int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks) + if marks != 0 { + t.Errorf("Expected the finish to clear the mark, %d left", marks) + } + if err := q.CancelJob("run"); !errors.Is(err, contract.ErrJobEnded) { + t.Errorf("CancelJob of an ended job: got %v, want ErrJobEnded", err) + } +} + +// A marked waiting job is never handed out, by any pop: it is dropped with +// outcome cancelled and the pop goes on with the next item. +func TestCancelWaiting(t *testing.T) { + q, qs := openQueue(t, WithRemoveOnComplete(false)) + defer qs.Close() + + for _, id := range []string{"a", "b", "c", "d", "e"} { + q.Enqueue(job(id)) + } + q.EnqueueOnce([]byte("raw"), "f") // a Job ID from EnqueueOnce, not JSON + q.Enqueue([]byte("g")) + chA, chF := watch(q, "a"), watch(q, "f") + for _, id := range []string{"a", "c", "d", "f"} { + if err := q.CancelJob(id); err != nil { + t.Fatal(err) + } + } + + if v, _ := pop(t, q); v != string(job("b")) { + t.Errorf("DequeueWithAckId: got %s, want b", v) + } + items, acks := make([]any, 2), make([]string, 2) + n, _, err := q.DequeueBatchWithAckIds(items, acks) + if err != nil || n != 2 || string(items[0].([]byte)) != string(job("e")) || string(items[1].([]byte)) != "g" { + t.Errorf("DequeueBatchWithAckIds: got %d %v %v", n, err, items[:n]) + } + if got := waitOutcome(t, chA); got.State != contract.JobCancelled || got.Error != "cancelled" { + t.Errorf("got %+v", got) + } + if got := waitOutcome(t, chF); got.State != contract.JobCancelled { + t.Errorf("got %+v", got) + } + if n := countRows(t, q.f.db, "status = 'cancelled'"); n != 4 { + t.Errorf("Expected 4 kept cancelled rows, got %d", n) + } + if _, ok, _ := q.DequeueWithAckId(); ok { + t.Error("Expected nothing left") + } + + // Dequeue without ack drops too + q.Enqueue(job("h")) + q.Enqueue(job("i")) + q.CancelJob("h") + if v, ok := q.Dequeue(); !ok || string(v.([]byte)) != string(job("i")) { + t.Errorf("Dequeue: got %v %v, want i", v, ok) + } + if got := waitOutcome(t, watch(q, "h")); got.State != contract.JobCancelled { + t.Errorf("got %+v", got) + } +} + +// EnqueueOnce that stores an id drops its old outcome and its cancel mark +// (spec 6.13, 6.14 Re-add); a dedupe hit changes neither. +func TestReaddClearsOutcomeAndMark(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + q.EnqueueOnce(job("r"), "r") + _, ack := pop(t, q) + q.Acknowledge(ack) + if countJobs(t, q) != 1 { + t.Fatal("Expected a record") + } + q.EnqueueOnce(job("r"), "r") + if countJobs(t, q) != 0 { + t.Error("Expected the re-add to drop the old record") + } + ch := watch(q, "r") + noOutcome(t, ch) // the earlier run is not reported + + if err := q.CancelJob("r"); err != nil { + t.Fatal(err) + } + q.EnqueueOnce(job("r"), "r") // dedupe hit: the row is still there + if _, ok, _ := q.DequeueWithAckId(); ok { + t.Fatal("Expected the marked job dropped") + } + if got := waitOutcome(t, ch); got.State != contract.JobCancelled { + t.Errorf("got %+v", got) + } + + q.CancelJob("s") // unknown id: marked + q.EnqueueOnce(job("s"), "s") + if v, _ := pop(t, q); v != string(job("s")) { + t.Errorf("Expected the re-added job handed out, got %s", v) + } +} + +// Purge drops outcome records and cancel marks with the items. +func TestPurgeClearsOutcomesAndMarks(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.Enqueue(job("done")) + _, ack := pop(t, q) + q.Acknowledge(ack) + q.CancelJob("later") + q.Purge() + var marks int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks) + if countJobs(t, q) != 0 || marks != 0 { + t.Errorf("Expected Purge to clear records and marks, got %d and %d", countJobs(t, q), marks) + } + q.Enqueue(job("later")) + if v, _ := pop(t, q); v != string(job("later")) { + t.Errorf("Expected the purged mark gone, got %s", v) + } +} + +// OnCancel reports marks made after it, not before; stop ends it; an +// expired mark neither reaches hooks nor drops items. +func TestOnCancel(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.CancelJob("old") + ids := make(chan string, 4) + stop := q.OnCancel(func(id string) { ids <- id }) + q.CancelJob("new") + select { + case id := <-ids: + if id != "new" { + t.Errorf("got %q, want new", id) + } + case <-time.After(5 * time.Second): + t.Fatal("OnCancel did not fire") + } + stop() + q.CancelJob("after") + select { + case id := <-ids: + t.Errorf("got %q after stop", id) + case <-time.After(4 * pollEvery): + } + + q.f.db.Exec("UPDATE "+q.cancelled+" SET expires_at = ? WHERE job_id = 'old'", time.Now().Add(-time.Second).UnixMilli()) + q.Enqueue(job("old")) + if v, _ := pop(t, q); v != string(job("old")) { + t.Errorf("Expected an expired mark to drop nothing, got %s", v) + } + if err := q.CancelJob(""); err == nil { + t.Error("Expected an empty id to fail") + } +} + +// Retention trims each state group by age and count. +func TestOutcomeRetention(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + now := time.Now() + tx, _ := q.f.db.Begin() + for i := range keepCompleted + 5 { + tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES (?, 'completed', ?)", "c"+strconv.Itoa(i), now.UnixMilli()+int64(i)) + } + tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", now.Add(-keepFailedAge).UnixMilli()) + tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'cancelled', ?)", now.UnixMilli()) + if err := q.trim(tx, now); err != nil { + t.Fatal(err) + } + tx.Commit() + if n := countJobs(t, q); n != keepCompleted+1 { + t.Errorf("Expected %d records, got %d", keepCompleted+1, n) + } + var oldest string + q.f.db.QueryRow("SELECT job_id FROM " + q.jobs + " WHERE state = 'completed' ORDER BY finished_at LIMIT 1").Scan(&oldest) + if oldest != "c5" { + t.Errorf("Expected the 5 oldest completed trimmed, oldest left %s", oldest) + } +} + +// varmq end to end: a result job added on one queue object runs on a +// worker bound to another (another process), and its handle gets the +// result; Cancel stops a running job and drops a waiting one. +func TestVarmqRemoteResultAndCancel(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q2 := otherProcess(t, q) + + started := make(chan struct{}) + ran := make(chan int, 8) + w := varmq.NewResultWorker(func(j varmq.Job[int]) (int, error) { + ran <- j.Data() + if j.Data() < 0 { + close(started) + <-j.Context().Done() + return 0, j.Context().Err() + } + return j.Data() * 2, nil + }) + defer w.Stop() + + p := varmq.NewResultQueue[int, int](q) + waiting, ok := p.Add(7) // no worker yet + if !ok { + t.Fatal("Add failed") + } + if err := waiting.Cancel(); err != nil { + t.Fatal(err) + } + w.WithQueue(q2) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + j, _ := p.Add(21) + if err := j.WaitContext(ctx); err != nil { + t.Fatal(err) + } + if r, err := j.Result(); err != nil || r != 42 { + t.Errorf("Result: got %d %v, want 42", r, err) + } + if err := waiting.WaitContext(ctx); err != nil { + t.Fatal(err) + } + if _, err := waiting.Result(); !errors.Is(err, varmq.ErrJobCancelled) { + t.Errorf("waiting job: got %v, want ErrJobCancelled", err) + } + + running, _ := p.Add(-1) + select { + case <-started: + case <-ctx.Done(): + t.Fatal("job did not start") + } + if err := running.Cancel(); err != nil { + t.Fatal(err) + } + if err := running.WaitContext(ctx); err != nil { + t.Fatal(err) + } + if _, err := running.Result(); !errors.Is(err, varmq.ErrJobCancelled) { + t.Errorf("running job: got %v, want ErrJobCancelled", err) + } + close(ran) + for v := range ran { + if v == 7 { + t.Error("The cancelled waiting job ran") + } + } +} diff --git a/registry.go b/registry.go index 24aeb0e..517ef07 100644 --- a/registry.go +++ b/registry.go @@ -62,8 +62,11 @@ func driverName(pages int) string { // dsn returns the connection string for a database file: WAL, relaxed // fsync (safe under WAL) and a busy timeout so writers wait for each other. +// Transactions begin IMMEDIATE: one that reads before it writes (CancelJob, +// a gated pop) would otherwise fail with SQLITE_BUSY, not wait, when another +// process wrote in between. func dsn(path string) string { - return path + "?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000" + return path + "?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000&_txlock=immediate" } // registry holds every open file in the process, keyed by resolved path. @@ -121,6 +124,10 @@ func (f *file) release() error { if registry.files[f.key] == f { delete(registry.files, f.key) } + // Stop the queues' polls before the pool closes under them + for _, h := range f.handles { + h.queue().Close() + } registry.mu.Unlock() // Close waits for running queries, so it runs outside the lock From c51876e83fb0a80fa3897d92ed92f5dbc8205d93 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 21:30:05 +0600 Subject: [PATCH 03/10] docs: changelog for job outcome and cancel --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b5eabe..ee7fa9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is not stored. +- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error. +- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it every 50 ms, from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d. +- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per pop); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended. +- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks. + +### Changed + +- Requires Go 1.27 (varmq dependency). Transactions begin `IMMEDIATE`. A finish (`Acknowledge`, `Reject`) always runs in a transaction; a pop runs in one while any cancel mark is live. ## [0.3.0] - 2026-10-23 From 5ea8a0853d33d7176ce03dafbdeac12824f0efb3 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 21:32:11 +0600 Subject: [PATCH 04/10] fix: cancel poll era, report items left at drop cap A poll from before an OnCancel reset no longer advances lastSeq past the new hook's marks. A batch pop stopped at maxDrops reports one item left so the caller pops again. Go 1.27 under Breaking. --- CHANGELOG.md | 6 +++++- cancel.go | 10 +++++++--- outcome.go | 5 +++-- queue.go | 51 +++++++++++++++++++++++++++++---------------------- queue_test.go | 24 ++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee7fa9c..a32885c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,9 +14,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per pop); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended. - `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks. +### Breaking + +- Requires Go 1.27 (was 1.24), for the varmq dependency. + ### Changed -- Requires Go 1.27 (varmq dependency). Transactions begin `IMMEDIATE`. A finish (`Acknowledge`, `Reject`) always runs in a transaction; a pop runs in one while any cancel mark is live. +- Transactions begin `IMMEDIATE`. A finish (`Acknowledge`, `Reject`) always runs in a transaction; a pop runs in one while any cancel mark is live. ## [0.3.0] - 2026-10-23 diff --git a/cancel.go b/cancel.go index ce8f1c9..e51d23f 100644 --- a/cancel.go +++ b/cancel.go @@ -107,6 +107,7 @@ func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) { } if len(w.hooks) == 0 { w.lastSeq = last.Int64 + w.era++ } if w.hooks == nil { w.hooks = map[*func(string)]struct{}{} @@ -121,8 +122,9 @@ func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) { } } -// pollCancels passes the live marks newer than seq to hooks. -func (q *Queue) pollCancels(seq int64, hooks []func(string)) { +// pollCancels passes the live marks newer than seq to hooks, a snapshot +// taken in era. +func (q *Queue) pollCancels(seq int64, era int, hooks []func(string)) { rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? AND expires_at > ? ORDER BY seq LIMIT %d", q.cancelled, maxDrops), seq, time.Now().UnixMilli()) if err != nil { @@ -142,7 +144,9 @@ func (q *Queue) pollCancels(seq int64, hooks []func(string)) { w := &q.w w.mu.Lock() - w.lastSeq = max(w.lastSeq, seq) + if w.era == era { + w.lastSeq = max(w.lastSeq, seq) + } w.mu.Unlock() for _, id := range ids { for _, fn := range hooks { diff --git a/outcome.go b/outcome.go index 5aa91e4..dd64287 100644 --- a/outcome.go +++ b/outcome.go @@ -164,6 +164,7 @@ type watchers struct { byID map[string]map[*outcomeWatch]struct{} hooks map[*func(string)]struct{} lastSeq int64 // newest cancel mark seq OnCancel has seen + era int // bumped when OnCancel resets lastSeq; a poll from an older era must not advance it polling bool closed bool stop chan struct{} // closed by close @@ -264,14 +265,14 @@ func (q *Queue) poll() { for fn := range w.hooks { hooks = append(hooks, *fn) } - seq := w.lastSeq + seq, era := w.lastSeq, w.era w.mu.Unlock() if len(ids) > 0 { q.catchUp(ids) } if len(hooks) > 0 { - q.pollCancels(seq, hooks) + q.pollCancels(seq, era, hooks) } } } diff --git a/queue.go b/queue.go index 471326f..fded499 100644 --- a/queue.go +++ b/queue.go @@ -364,12 +364,12 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { // claimN moves up to n pending rows to processing in one statement and // returns them in queue order -func (q *Queue) claimN(n int) ([]claimed, error) { +func (q *Queue) claimN(n int) ([]claimed, bool, error) { q.f.mu.Lock() - out, err := q.claimGated(n) + out, capped, err := q.claimGated(n) q.f.mu.Unlock() if err != nil { - return nil, err + return nil, false, err } // RETURNING order is unspecified @@ -379,33 +379,35 @@ func (q *Queue) claimN(n int) ([]claimed, error) { } return cmp.Compare(a.id, b.id) }) - return out, nil + return out, capped, nil } // claimGated claims up to n rows; with a live cancel mark on the file it // runs in one transaction that drops each marked row it claims (outcome -// cancelled) and claims again, at most maxDrops drops. The caller holds the -// write mutex. -func (q *Queue) claimGated(n int) ([]claimed, error) { +// cancelled) and claims again, at most maxDrops drops; capped reports that +// it stopped there. The caller holds the write mutex. +func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) { now := time.Now() if !q.anyMark(now) { - return q.claimRows(q.claim, n) + out, err = q.claimRows(q.claim, n) + return out, false, err } tx, err := q.f.db.Begin() if err != nil { - return nil, err + return nil, false, err } defer tx.Rollback() claim := tx.Stmt(q.claim) - out := make([]claimed, 0, n) - for drops := 0; len(out) < n && drops < maxDrops; { + out = make([]claimed, 0, n) + drops := 0 + for len(out) < n && drops < maxDrops { rows, err := q.claimRows(claim, n-len(out)) - if err != nil || len(rows) == 0 { - if err != nil { - return nil, err - } + if err != nil { + return nil, false, err + } + if len(rows) == 0 { break } // Every claimed row is handed out or dropped, so a batch may pass the cap @@ -413,19 +415,19 @@ func (q *Queue) claimGated(n int) ([]claimed, error) { id := jobIDOf(c.once, c.data) drop, err := q.marked(tx, id, now) if err != nil { - return nil, err + return nil, false, err } if !drop { out = append(out, c) continue } if _, err := q.end(tx, c.id, id, contract.JobCancelled, "cancelled", nil, now); err != nil { - return nil, err + return nil, false, err } drops++ } } - return out, tx.Commit() + return out, len(out) < n && drops >= maxDrops, tx.Commit() } // claimRows runs a claim statement and reads every row it returns; the @@ -512,7 +514,7 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) { return nil, false, "" } - c, err := q.claimN(1) + c, _, err := q.claimN(1) if err != nil || len(c) == 0 { return nil, false, "" } @@ -530,12 +532,17 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini return 0, 0, nil } - c, err := q.claimN(want) + c, capped, err := q.claimN(want) if err != nil { return 0, 0, err } + // At the drop cap items may wait still: report one, so the caller pops again + left := 0 + if capped { + left = 1 + } if len(c) == 0 { - return 0, 0, nil + return 0, left, nil } for i, row := range c { items[i], ackIDs[i] = row.data, strconv.FormatInt(row.id, 10) @@ -546,7 +553,7 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini if err := q.countCapped.QueryRow().Scan(&remaining); err != nil { return len(c), 1, nil } - return len(c), remaining, nil + return len(c), max(remaining, left), nil } // Acknowledge marks a processing item as completed, deleting it unless diff --git a/queue_test.go b/queue_test.go index 68775ff..680c3e5 100644 --- a/queue_test.go +++ b/queue_test.go @@ -1340,3 +1340,27 @@ func TestVarmqRemoteResultAndCancel(t *testing.T) { } } } + +// A pop drops at most maxDrops marked items; at the cap it reports items +// left, and the next pop goes on. +func TestCancelDropCap(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + for i := range maxDrops { + id := "m" + strconv.Itoa(i) + q.Enqueue(job(id)) + q.CancelJob(id) + } + q.Enqueue([]byte("plain")) + + items, acks := make([]any, 1), make([]string, 1) + if n, left, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != 0 || left != 1 { + t.Fatalf("at the cap: got n=%d left=%d err=%v, want 0, 1", n, left, err) + } + if v, _ := pop(t, q); v != "plain" { + t.Errorf("got %s, want plain", v) + } + if n := countJobs(t, q); n != maxDrops { + t.Errorf("Expected %d cancelled records, got %d", maxDrops, n) + } +} From 9374f80d550fffc8b7ee13d0206409475759bd84 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 21:35:44 +0600 Subject: [PATCH 05/10] chore: Go 1.27 in CI and README, commitRound doc go.mod needs 1.27 now. Note that a multi-statement op opens its own tx in a round of one. --- .github/workflows/sqliteq.yml | 2 +- README.md | 4 ++-- commit.go | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sqliteq.yml b/.github/workflows/sqliteq.yml index b658b4d..bf9d168 100644 --- a/.github/workflows/sqliteq.yml +++ b/.github/workflows/sqliteq.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24.1" + go-version: "1.27.0" cache: true - name: Install dependencies diff --git a/README.md b/README.md index d911546..6058b8b 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Go Reference CI Codecov - Go Version + Go Version License

@@ -49,7 +49,7 @@ go get github.com/goptics/sqliteq ``` -Requires Go 1.24+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler. +Requires Go 1.27+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler. ## Quick Start diff --git a/commit.go b/commit.go index edb4f29..be6fed2 100644 --- a/commit.go +++ b/commit.go @@ -10,7 +10,7 @@ import ( // code (such as a driver.Valuer): a panic there would strand the round and // leave its transaction open. Convert user values before building it. type op struct { - run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit + run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit, or a tx of its own done chan result // buffered 2: one lead wake-up and the outcome } @@ -97,6 +97,7 @@ func (b *batcher) lead() { // transaction of its own, so one bad call fails alone. A round of one op runs // with a nil tx, as one autocommit statement: a database/sql Tx costs a // context and a goroutine, which a round with nothing to share does not repay. +// An op of several statements (a finish) opens its own tx then. func (f *file) commitRound(ops []*op) { f.mu.Lock() defer f.mu.Unlock() From 7cc1fd646d48fcf039194ed810539049d38534c8 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 22:06:17 +0600 Subject: [PATCH 06/10] fix: review findings on outcome and cancel - Outcome poll reads records by seq cursor (AUTOINCREMENT), not a 500-id sample: a burst of finishes no longer loses watches. - Count trim spares records under 1 min old; first record of a queue object trims; poll sweeps retention + expired marks each minute. Index on marks expires_at. - OnCancel: per-hook from seq, hooks picked after the read: a hook added mid-poll gets its marks, stopped or closed ones get none. A failed first read gives a no-op hook, not a 24 h replay. - deliver reaches only watches started before the read (re-add). - Drop cap per tx: claims never pass it; single pops go on past it. - Queue names must not contain __sqliteq_. --- CHANGELOG.md | 5 +- cancel.go | 68 +++++++++++-------- outcome.go | 185 +++++++++++++++++++++++++++++++++++++++------------ queue.go | 54 ++++++++++----- registry.go | 4 ++ 5 files changed, 226 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a32885c..8961d18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error. -- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it every 50 ms, from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d. -- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per pop); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended. +- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it by seq every 50 ms (new records only), from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d; the count limit spares records under 1 min old. A polling queue object also sweeps retention and expired marks once a minute. +- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per transaction; a single-item pop goes on past that, a batch pop reports an item left); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended. - `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks. ### Breaking - Requires Go 1.27 (was 1.24), for the varmq dependency. +- A queue name must not contain `__sqliteq_`, in both layouts: a queue's indexes, sidecar tables and trigger use it. ### Changed diff --git a/cancel.go b/cancel.go index e51d23f..6b4445e 100644 --- a/cancel.go +++ b/cancel.go @@ -92,13 +92,16 @@ func (q *Queue) AcknowledgeCancelled(ackID, reason string) bool { // cancel request made after the call, through any queue object on this // database file, until stop. Marks are polled every pollEvery; each call // runs on its own goroutine. A mark gone by the poll (the job finished or -// was re-added) is never reported. A closed queue reports nothing. +// was re-added) is never reported. A closed queue, or a failed first read, +// reports nothing. func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) { // Read before the lock: a mark made after this read has a higher seq var last sql.NullInt64 - _ = q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.cancelled)).Scan(&last) + if err := q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.cancelled)).Scan(&last); err != nil { + return func() {} // 0 would replay every live mark + } - p := &fn + h := &cancelHook{fn: fn, from: last.Int64} w := &q.w w.mu.Lock() defer w.mu.Unlock() @@ -106,51 +109,63 @@ func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) { return func() {} } if len(w.hooks) == 0 { - w.lastSeq = last.Int64 - w.era++ + w.lastSeq = h.from // no poll read marks for anyone since } if w.hooks == nil { - w.hooks = map[*func(string)]struct{}{} + w.hooks = map[*cancelHook]struct{}{} } - w.hooks[p] = struct{}{} + w.hooks[h] = struct{}{} w.startLocked(q) return func() { w.mu.Lock() defer w.mu.Unlock() - delete(w.hooks, p) + delete(w.hooks, h) } } -// pollCancels passes the live marks newer than seq to hooks, a snapshot -// taken in era. -func (q *Queue) pollCancels(seq int64, era int, hooks []func(string)) { +// cancelHook is one OnCancel callback; it gets marks with seq above from. +type cancelHook struct { + fn func(string) + from int64 +} + +// pollCancels reads the live marks newer than seq and passes each to the +// hooks live after the read whose from is below it, so a hook added during +// the read still gets its marks, and a stopped one gets none. +func (q *Queue) pollCancels(seq int64) { rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? AND expires_at > ? ORDER BY seq LIMIT %d", q.cancelled, maxDrops), seq, time.Now().UnixMilli()) if err != nil { return } - var ids []string + type mark struct { + seq int64 + id string + } + var marks []mark for rows.Next() { - var id string - if rows.Scan(&seq, &id) != nil { - continue + var m mark + if rows.Scan(&m.seq, &m.id) == nil { + marks = append(marks, m) } - ids = append(ids, id) } - if rows.Close() != nil || rows.Err() != nil { + if rows.Close() != nil || rows.Err() != nil || len(marks) == 0 { return // read again from the old seq } w := &q.w w.mu.Lock() - if w.era == era { - w.lastSeq = max(w.lastSeq, seq) + defer w.mu.Unlock() + if w.closed { + return } - w.mu.Unlock() - for _, id := range ids { - for _, fn := range hooks { - go fn(id) + w.lastSeq = max(w.lastSeq, marks[len(marks)-1].seq) + for h := range w.hooks { + for _, m := range marks { + if m.seq > h.from { + go h.fn(m.id) + } } } } @@ -165,12 +180,13 @@ func (q *Queue) anyMark(now time.Time) bool { return n == 1 } -// marked reports whether job id has a live mark, in tx. -func (q *Queue) marked(tx *sql.Tx, id string, now time.Time) (bool, error) { +// marked reports whether job id has a live mark; isMarked is q.isMarked +// in the caller's tx. +func marked(isMarked *sql.Stmt, id string, now time.Time) (bool, error) { if id == "" { return false, nil } var n int - err := tx.Stmt(q.isMarked).QueryRow(id, now.UnixMilli()).Scan(&n) + err := isMarked.QueryRow(id, now.UnixMilli()).Scan(&n) return n == 1, err } diff --git a/outcome.go b/outcome.go index dd64287..643f2a3 100644 --- a/outcome.go +++ b/outcome.go @@ -14,9 +14,11 @@ import ( // Job outcome (varmq spec/redis-storage.md 1.6.0, 6.13), on SQLite. Each // finish of a job with a Job ID writes one row to __sqliteq_jobs in -// the finish's transaction. SQLite has no pub/sub, so a watcher is -// answered by polling that table: one goroutine per queue object, alive -// only while it has watches or OnCancel hooks. +// the finish's transaction, under a new AUTOINCREMENT seq. SQLite has no +// pub/sub, so a watcher is answered by polling that table by seq (records +// newer than the last read): one goroutine per queue object, alive only +// while it has watches or OnCancel hooks. It also sweeps retention and +// expired marks. var ( _ contract.OutcomeWatcher = (*Queue)(nil) @@ -29,9 +31,12 @@ const ( // pollEvery is how often a queue object with watches or hooks reads // the outcome and cancel tables. pollEvery = 50 * time.Millisecond - // maxCatchUp caps the watched ids one poll reads; map order rotates a - // larger set. - maxCatchUp = 500 + // pollBatch caps the rows one cursor read returns; a poll reads again + // until it gets fewer. + pollBatch = 1000 + // sweepEvery is how often a polling queue object trims retention and + // drops expired marks. + sweepEvery = time.Minute // Retention (spec 5 defaults): cancelled records count as failed. keepCompleted = 1000 @@ -40,6 +45,9 @@ const ( keepFailedAge = 7 * 24 * time.Hour // trimEvery is how many records a queue object writes between trims. trimEvery = 100 + // trimFloor: the count limit never drops a record younger than this, + // so a burst can't trim one before a poll reads it. + trimFloor = time.Minute jobsSuffix = "__sqliteq_jobs" cancelledSuffix = "__sqliteq_cancelled" @@ -52,28 +60,31 @@ const ( // holds the write mutex. func (q *Queue) initSidecars() error { for _, s := range []string{ + // AUTOINCREMENT: seq never reuses a deleted row's, so a seq > last + // cursor sees every new record and mark; REPLACE gives a new seq `CREATE TABLE IF NOT EXISTS %[2]s ( - job_id TEXT PRIMARY KEY, + seq INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL UNIQUE, state TEXT NOT NULL, error TEXT, result BLOB, finished_at INTEGER NOT NULL - ) WITHOUT ROWID`, + )`, `CREATE INDEX IF NOT EXISTS %[4]s ON %[2]s (finished_at)`, - // AUTOINCREMENT: seq never reuses a deleted mark's, so OnCancel's - // seq > last cursor sees every new mark `CREATE TABLE IF NOT EXISTS %[3]s ( seq INTEGER PRIMARY KEY AUTOINCREMENT, job_id TEXT NOT NULL UNIQUE, expires_at INTEGER NOT NULL )`, + `CREATE INDEX IF NOT EXISTS %[6]s ON %[3]s (expires_at)`, `CREATE TRIGGER IF NOT EXISTS %[5]s AFTER INSERT ON %[1]s WHEN NEW.job_id IS NOT NULL BEGIN DELETE FROM %[2]s WHERE job_id = NEW.job_id; DELETE FROM %[3]s WHERE job_id = NEW.job_id; END`, } { query := fmt.Sprintf(s, q.table, q.jobs, q.cancelled, - quoteIdent(q.name+jobsSuffix+"_finished"), quoteIdent(q.name+readdSuffix)) + quoteIdent(q.name+jobsSuffix+"_finished"), quoteIdent(q.name+readdSuffix), + quoteIdent(q.name+cancelledSuffix+"_expires")) if _, err := q.f.db.Exec(query); err != nil { return err } @@ -113,14 +124,16 @@ func (q *Queue) record(tx *sql.Tx, id string, state contract.JobState, reason st if _, err := tx.Stmt(q.unmark).Exec(id); err != nil { return err } - if q.records.Add(1)%trimEvery == 0 { + // The first record of a queue object trims too: a short-lived process + // may write fewer than trimEvery + if q.records.Add(1)%trimEvery == 1 { return q.trim(tx, now) } return nil } // trim applies retention: per state group, records past the age limit and -// those past the newest N. +// those past the newest N that are older than trimFloor. func (q *Queue) trim(tx *sql.Tx, now time.Time) error { for _, g := range []struct { where string @@ -134,8 +147,8 @@ func (q *Queue) trim(tx *sql.Tx, now time.Time) error { now.Add(-g.age).UnixMilli()); err != nil { return err } - if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %[1]s WHERE job_id IN (SELECT job_id FROM %[1]s WHERE %[2]s ORDER BY finished_at DESC LIMIT -1 OFFSET %[3]d)", - q.jobs, g.where, g.keep)); err != nil { + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %[1]s WHERE finished_at <= ? AND job_id IN (SELECT job_id FROM %[1]s WHERE %[2]s ORDER BY finished_at DESC LIMIT -1 OFFSET %[3]d)", + q.jobs, g.where, g.keep), now.Add(-trimFloor).UnixMilli()); err != nil { return err } } @@ -154,6 +167,7 @@ func (q *Queue) AcknowledgeResult(ackID string, result []byte) bool { type outcomeWatch struct { fn func(contract.JobOutcome) + gen uint64 // w.gen at its start done atomic.Bool // called or cancelled } @@ -162,9 +176,10 @@ type outcomeWatch struct { type watchers struct { mu sync.Mutex byID map[string]map[*outcomeWatch]struct{} - hooks map[*func(string)]struct{} - lastSeq int64 // newest cancel mark seq OnCancel has seen - era int // bumped when OnCancel resets lastSeq; a poll from an older era must not advance it + hooks map[*cancelHook]struct{} + lastSeq int64 // newest cancel mark seq a poll has read + outSeq int64 // newest outcome record seq a poll has read + gen uint64 // watches started; a read reaches only watches started before it polling bool closed bool stop chan struct{} // closed by close @@ -209,6 +224,23 @@ func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel fu w.mu.Unlock() return func() {} } + if len(w.byID) == 0 { + // No poll moved the cursor while nothing was watched: start it at + // the newest record. Older ones are this watch's catchUp below. + w.mu.Unlock() + var last sql.NullInt64 + err := q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.jobs)).Scan(&last) + w.mu.Lock() + if w.closed { + w.mu.Unlock() + return func() {} + } + if err == nil && len(w.byID) == 0 { + w.outSeq = last.Int64 + } + } + w.gen++ + o.gen = w.gen if w.byID == nil { w.byID = map[string]map[*outcomeWatch]struct{}{} } @@ -219,7 +251,7 @@ func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel fu w.startLocked(q) w.mu.Unlock() - go q.catchUp([]string{id}) + go q.catchUp([]string{id}, o.gen) return func() { if o.done.CompareAndSwap(false, true) { @@ -235,12 +267,13 @@ func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel fu } } -// poll reads the watched outcomes and new cancel marks every pollEvery, -// until Close or until nothing is watched. +// poll reads new outcome records and cancel marks every pollEvery, and +// sweeps every sweepEvery, until Close or until nothing is watched. func (q *Queue) poll() { t := time.NewTicker(pollEvery) defer t.Stop() w := &q.w + swept := time.Now() for { select { case <-w.stop: @@ -254,32 +287,85 @@ func (q *Queue) poll() { w.mu.Unlock() return } - ids := make([]string, 0, min(len(w.byID), maxCatchUp)) - for id := range w.byID { - if len(ids) == maxCatchUp { - break + watching, hooks := len(w.byID) > 0, len(w.hooks) > 0 + outSeq, seq, gen := w.outSeq, w.lastSeq, w.gen + w.mu.Unlock() + + if watching { + q.pollOutcomes(outSeq, gen) + } + if hooks { + q.pollCancels(seq) + } + if time.Since(swept) >= sweepEvery { + swept = time.Now() + q.sweep() + } + } +} + +// pollOutcomes reads the records newer than seq, in pollBatch reads, and +// delivers the watched ones to watches up to gen. +func (q *Queue) pollOutcomes(seq int64, gen uint64) { + for { + rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? ORDER BY seq LIMIT %d", q.jobs, pollBatch), seq) + if err != nil { + return + } + var ids []string + for rows.Next() { + var id string + if rows.Scan(&seq, &id) == nil { + ids = append(ids, id) } - ids = append(ids, id) } - hooks := make([]func(string), 0, len(w.hooks)) - for fn := range w.hooks { - hooks = append(hooks, *fn) + if rows.Close() != nil || rows.Err() != nil || len(ids) == 0 { + return // read again from the old seq } - seq, era := w.lastSeq, w.era - w.mu.Unlock() - if len(ids) > 0 { - q.catchUp(ids) + w := &q.w + w.mu.Lock() + w.outSeq = max(w.outSeq, seq) + watched := ids[:0] + for _, id := range ids { + if w.byID[id] != nil { + watched = append(watched, id) + } + } + w.mu.Unlock() + if len(watched) > 0 { + q.catchUp(watched, gen) } - if len(hooks) > 0 { - q.pollCancels(seq, era, hooks) + if len(ids) < pollBatch { + return } } } -// catchUp delivers the ids whose outcome record exists. A failed read is -// retried by the next poll. -func (q *Queue) catchUp(ids []string) { +// sweep applies retention and drops expired cancel marks (spec 5, 6.14 +// Prune), so an idle queue ages out too. +func (q *Queue) sweep() { + q.f.mu.Lock() + defer q.f.mu.Unlock() + tx, err := q.f.db.Begin() + if err != nil { + return + } + defer tx.Rollback() + now := time.Now() + if q.trim(tx, now) != nil { + return + } + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE expires_at <= ?", q.cancelled), now.UnixMilli()); err != nil { + return + } + tx.Commit() +} + +// catchUp delivers the ids whose outcome record exists to their watches +// up to gen, those started before the read. A failed read is retried by +// the next poll. +func (q *Queue) catchUp(ids []string, gen uint64) { args := make([]any, len(ids)) for i, id := range ids { args[i] = id @@ -304,20 +390,31 @@ func (q *Queue) catchUp(ids []string) { rows.Close() for _, out := range outs { - q.deliver(out) + q.deliver(out, gen) } } -// deliver ends every watch of out.ID: each fn runs once, on its own -// goroutine, so a slow fn holds up no poll. -func (q *Queue) deliver(out contract.JobOutcome) { +// deliver ends the watches of out.ID started up to gen: each fn runs once, +// on its own goroutine, so a slow fn holds up no poll. A watch started +// after the read stays: the record read may be an earlier run's, which a +// re-add since deleted (spec 6.13 Re-add). +func (q *Queue) deliver(out contract.JobOutcome, gen uint64) { w := &q.w w.mu.Lock() set := w.byID[out.ID] - delete(w.byID, out.ID) + var ended []*outcomeWatch + for o := range set { + if o.gen <= gen { + ended = append(ended, o) + delete(set, o) + } + } + if len(set) == 0 { + delete(w.byID, out.ID) + } w.mu.Unlock() - for o := range set { + for _, o := range ended { go func() { if o.done.CompareAndSwap(false, true) { o.fn(out) diff --git a/queue.go b/queue.go index fded499..be1fbdd 100644 --- a/queue.go +++ b/queue.go @@ -399,21 +399,21 @@ func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) { } defer tx.Rollback() - claim := tx.Stmt(q.claim) + claim, isMarked := tx.Stmt(q.claim), tx.Stmt(q.isMarked) out = make([]claimed, 0, n) drops := 0 for len(out) < n && drops < maxDrops { - rows, err := q.claimRows(claim, n-len(out)) + // No more than the drops left: a batch never passes the cap + rows, err := q.claimRows(claim, min(n-len(out), maxDrops-drops)) if err != nil { return nil, false, err } if len(rows) == 0 { break } - // Every claimed row is handed out or dropped, so a batch may pass the cap for _, c := range rows { id := jobIDOf(c.once, c.data) - drop, err := q.marked(tx, id, now) + drop, err := marked(isMarked, id, now) if err != nil { return nil, false, err } @@ -471,33 +471,45 @@ func (q *Queue) Dequeue() (any, bool) { data, _, err := popRow(q.pop) return data, err == nil } + for { + data, capped, err := q.popGated(now) + if err != nil || data != nil || !capped { + return data, err == nil && data != nil + } + } +} - // Gated: a marked item is dropped (outcome cancelled) and the next one popped +// popGated pops in one transaction, dropping each marked item it pops +// (outcome cancelled), at most maxDrops; capped reports it stopped there +// with nothing. The caller holds the write mutex. +func (q *Queue) popGated(now time.Time) (data []byte, capped bool, err error) { tx, err := q.f.db.Begin() if err != nil { - return nil, false + return nil, false, err } defer tx.Rollback() - pop := tx.Stmt(q.pop) + pop, isMarked := tx.Stmt(q.pop), tx.Stmt(q.isMarked) for range maxDrops { data, once, err := popRow(pop) + if errors.Is(err, sql.ErrNoRows) { + return nil, false, tx.Commit() // keep the drops + } if err != nil { - break + return nil, false, err } id := jobIDOf(once, data) - drop, err := q.marked(tx, id, now) + drop, err := marked(isMarked, id, now) if err != nil { - return nil, false + return nil, false, err } if !drop { - return data, tx.Commit() == nil + return data, false, tx.Commit() } if err := q.record(tx, id, contract.JobCancelled, "cancelled", nil, now); err != nil { - return nil, false + return nil, false, err } } - tx.Commit() // keep the drops; an empty queue or the cap hands out nothing - return nil, false + return nil, true, tx.Commit() } // popRow runs a pop statement: the deleted row's data and job_id, or @@ -514,11 +526,17 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) { return nil, false, "" } - c, _, err := q.claimN(1) - if err != nil || len(c) == 0 { - return nil, false, "" + // A pop stopped at the drop cap has nothing yet: pop again, it can't + // report items left + for { + c, capped, err := q.claimN(1) + if err != nil || len(c) == 0 && !capped { + return nil, false, "" + } + if len(c) > 0 { + return c[0].data, true, strconv.FormatInt(c[0].id, 10) + } } - return c[0].data, true, strconv.FormatInt(c[0].id, 10) } // DequeueBatchWithAckIds implements varmq.IBatchAcknowledgeable: it moves up diff --git a/registry.go b/registry.go index 517ef07..5db083a 100644 --- a/registry.go +++ b/registry.go @@ -150,6 +150,10 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) } key := foldName(name) + // A queue's indexes, sidecar tables and trigger are named __sqliteq_* + if strings.Contains(key, "__sqliteq_") { + return nil, fmt.Errorf("sqliteq: queue name %q must not contain __sqliteq_", name) + } if h, ok := f.handles[key]; ok { if (h.pq != nil) != priority { return nil, fmt.Errorf("sqliteq: %q is already open as a %s", name, h.kind()) From e009366a192c7ec8c60ca7a2db6cf31c959c1072 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 22:06:17 +0600 Subject: [PATCH 07/10] test: outcome and cancel review gaps Batched finishes (shared and fallback round), Close with live watch and hook, hooks during a poll and stale marks, repeat cancel, gated Dequeue to empty, kept-row ends, deliver generation, 3000-watch burst, reserved names, sidecars on a v0.2 file, retention floor and first record trim. Closed-queue watch check no longer vacuous. --- commit_test.go | 50 +++++++++ migration_test.go | 6 ++ queue_test.go | 258 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 310 insertions(+), 4 deletions(-) diff --git a/commit_test.go b/commit_test.go index 6b1a7e1..54fa30d 100644 --- a/commit_test.go +++ b/commit_test.go @@ -491,3 +491,53 @@ func TestGroupCommitAfterClose(t *testing.T) { } waitAll(t, &wg, 10*time.Second) } + +// Finishes that share a round write their outcomes in its transaction, +// and again each in its own when a bad call fails the round. +func TestGroupFinishOutcomes(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "shared", true: "fallback"}[fail], func(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + for _, id := range []string{"a", "b", "c", "d"} { + q.Enqueue(job(id)) + } + items, acks := make([]any, 4), make([]string, 4) + if n, _, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != 4 { + t.Fatalf("got %d %v", n, err) + } + rest := []func() bool{ + func() bool { return q.Reject(acks[1], "r") }, + func() bool { return q.AcknowledgeResult(acks[2], []byte(`1`)) }, + func() bool { return q.AcknowledgeCancelled(acks[3], "c") }, + } + if fail { + rest = append(rest, func() bool { return q.Enqueue(nil) }) + } + got := oneRound(t, q.f, func() bool { return q.Acknowledge(acks[0]) }, rest) + for i, ok := range got[:4] { + if !ok { + t.Errorf("finish %d failed", i) + } + } + want := map[string]string{"a": "completed||", "b": "failed|r|", "c": "completed||1", "d": "cancelled|c|"} + rows, err := q.f.db.Query("SELECT job_id, state, COALESCE(error, ''), COALESCE(CAST(result AS TEXT), '') FROM " + q.jobs) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + n := 0 + for rows.Next() { + var id, state, e, r string + rows.Scan(&id, &state, &e, &r) + if g := state + "|" + e + "|" + r; g != want[id] { + t.Errorf("%s: got %s, want %s", id, g, want[id]) + } + n++ + } + if n != 4 { + t.Errorf("Expected 4 records, got %d", n) + } + }) + } +} diff --git a/migration_test.go b/migration_test.go index bc94204..be706f8 100644 --- a/migration_test.go +++ b/migration_test.go @@ -129,6 +129,12 @@ func TestMigrateV02File(t *testing.T) { if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { t.Errorf("Round %d: indexes = %v, want %v", round, got, want) } + for _, side := range []string{name + jobsSuffix, name + cancelledSuffix, name + readdSuffix} { + var n int + if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = ?", side).Scan(&n); err != nil || n != 1 { + t.Errorf("Round %d: expected %s on the old file, got %d %v", round, side, n, err) + } + } if round == 1 { // The two processing rows are pending again after recovery diff --git a/queue_test.go b/queue_test.go index 680c3e5..c12eb7d 100644 --- a/queue_test.go +++ b/queue_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "path/filepath" + "slices" "strconv" "sync" "sync/atomic" @@ -1052,8 +1053,11 @@ func TestWatchEndedBeforeAndCancel(t *testing.T) { noOutcome(t, ch) q.Close() - q.WatchOutcome("early", func(o contract.JobOutcome) { ch <- o })() + q.WatchOutcome("early", func(o contract.JobOutcome) { ch <- o }) noOutcome(t, ch) + if err := q.CancelJob("x"); !errors.Is(err, errClosed) { + t.Errorf("CancelJob on a closed queue: got %v, want errClosed", err) + } } // Two queue objects on one file, as two processes: each learns of jobs the @@ -1246,15 +1250,34 @@ func TestOnCancel(t *testing.T) { } } -// Retention trims each state group by age and count. +// Retention trims each state group by age and count; the count limit +// spares records younger than trimFloor; a queue object's first record +// trims. func TestOutcomeRetention(t *testing.T) { q, qs := openQueue(t) defer qs.Close() now := time.Now() + insert := func(tx *sql.Tx, prefix string, n int, at time.Time) { + for i := range n { + if _, err := tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES (?, 'completed', ?)", prefix+strconv.Itoa(i), at.UnixMilli()+int64(i)); err != nil { + t.Fatal(err) + } + } + } + tx, _ := q.f.db.Begin() - for i := range keepCompleted + 5 { - tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES (?, 'completed', ?)", "c"+strconv.Itoa(i), now.UnixMilli()+int64(i)) + insert(tx, "fresh", keepCompleted+5, now) + if err := q.trim(tx, now); err != nil { + t.Fatal(err) } + tx.Commit() + if n := countJobs(t, q); n != keepCompleted+5 { + t.Fatalf("Expected records younger than trimFloor kept, got %d", n) + } + q.f.db.Exec("DELETE FROM " + q.jobs) + + tx, _ = q.f.db.Begin() + insert(tx, "c", keepCompleted+5, now.Add(-2*trimFloor)) tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", now.Add(-keepFailedAge).UnixMilli()) tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'cancelled', ?)", now.UnixMilli()) if err := q.trim(tx, now); err != nil { @@ -1269,6 +1292,17 @@ func TestOutcomeRetention(t *testing.T) { if oldest != "c5" { t.Errorf("Expected the 5 oldest completed trimmed, oldest left %s", oldest) } + + q2 := otherProcess(t, q) // records counter at 0 + q2.f.db.Exec("UPDATE "+q.jobs+" SET finished_at = ? WHERE job_id = 'c5'", now.Add(-keepCompletedAge).UnixMilli()) + q2.Enqueue(job("one")) + _, ack := pop(t, q2) + q2.Acknowledge(ack) + var n int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.jobs + " WHERE job_id = 'c5'").Scan(&n) + if n != 0 { + t.Error("Expected a queue object's first record to trim") + } } // varmq end to end: a result job added on one queue object runs on a @@ -1364,3 +1398,219 @@ func TestCancelDropCap(t *testing.T) { t.Errorf("Expected %d cancelled records, got %d", maxDrops, n) } } + +// Queues.Close stops polls and drops live watches and hooks without +// calling them. +func TestCloseDropsWatchesAndHooks(t *testing.T) { + q, qs := openQueue(t) + q2 := otherProcess(t, q) + ch := watch(q, "w") + ids := make(chan string, 1) + q.OnCancel(func(id string) { ids <- id }) + if err := qs.Close(); err != nil { + t.Fatal(err) + } + select { + case <-q.w.stop: + default: + t.Fatal("Expected Close to stop the poll") + } + q2.Enqueue(job("w")) + _, ack := pop(t, q2) + q2.Acknowledge(ack) + q2.CancelJob("c") + noOutcome(t, ch) + select { + case id := <-ids: + t.Errorf("hook called after Close with %q", id) + default: + } + if stop := q.OnCancel(func(string) { t.Error("hook on a closed queue") }); stop == nil { + t.Error("Expected a stop func") + } +} + +// A hook added while a poll reads gets the marks made after its call; a +// stopped hook gets none; a mark whose job finished (stale) reaches nobody. +func TestOnCancelHooksDuringPoll(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + a, b := make(chan string, 4), make(chan string, 4) + stopA := q.OnCancel(func(id string) { a <- id }) + q.w.mu.Lock() + seq := q.w.lastSeq // a poll's snapshot, taken before B + q.w.mu.Unlock() + q.CancelJob("before-b") + stopB := q.OnCancel(func(id string) { b <- id }) + q.CancelJob("x") + q.pollCancels(seq) + + // The queue's own poll may report a mark too: compare distinct ids + got := func(ch chan string) []string { + var ids []string + for { + select { + case id := <-ch: + ids = append(ids, id) + case <-time.After(4 * pollEvery): + slices.Sort(ids) + return slices.Compact(ids) + } + } + } + if ids := got(a); !slices.Equal(ids, []string{"before-b", "x"}) { + t.Errorf("hook A: got %v", ids) + } + if ids := got(b); !slices.Equal(ids, []string{"x"}) { + t.Errorf("hook B: got %v, want only the mark after its call", ids) + } + + var xSeq int64 + q.f.db.QueryRow("SELECT seq FROM " + q.cancelled + " WHERE job_id = 'x'").Scan(&xSeq) + stopA() + q.Enqueue(job("y")) + _, ack := pop(t, q) + q.CancelJob("y") + q.Acknowledge(ack) // the finish drops the mark: stale + q.CancelJob("z") + q.pollCancels(xSeq) + if ids := got(a); len(ids) != 0 { + t.Errorf("stopped hook got %v", ids) + } + if ids := got(b); !slices.Equal(ids, []string{"z"}) { + t.Errorf("hook B: got %v, want z only (y is stale)", ids) + } + stopB() +} + +// A repeat CancelJob gets a new seq and expiry, so hooks hear it again. +func TestCancelRepeatRenotifies(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + ids := make(chan string, 4) + defer q.OnCancel(func(id string) { ids <- id })() + read := func() (seq, exp int64) { + q.f.db.QueryRow("SELECT seq, expires_at FROM "+q.cancelled+" WHERE job_id = 'x'").Scan(&seq, &exp) + return + } + for i := range 2 { + q.CancelJob("x") + select { + case <-ids: + case <-time.After(5 * time.Second): + t.Fatalf("cancel %d not reported", i+1) + } + if i == 0 { + s1, e1 := read() + time.Sleep(2 * time.Millisecond) + defer func() { + if s2, e2 := read(); s2 <= s1 || e2 <= e1 { + t.Errorf("Expected a new seq and expiry: %d,%d -> %d,%d", s1, e1, s2, e2) + } + }() + } + } +} + +// A gated Dequeue that drops the last item returns empty and keeps the +// drop; the mark is gone after it. +func TestDequeueGatedEmpty(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.Enqueue(job("x")) + q.CancelJob("x") + ch := watch(q, "x") + if v, ok := q.Dequeue(); ok { + t.Fatalf("got %v", v) + } + if got := waitOutcome(t, ch); got.State != contract.JobCancelled { + t.Errorf("got %+v", got) + } + var marks int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks) + if q.Len() != 0 || marks != 0 { + t.Errorf("Expected no item and no mark, got %d and %d", q.Len(), marks) + } +} + +// With WithRemoveOnComplete(false) each end keeps its row with its status, +// and the record keeps the result. +func TestKeptRowEnds(t *testing.T) { + q, qs := openQueue(t, WithRemoveOnComplete(false)) + defer qs.Close() + q.Enqueue(job("r")) + q.Enqueue(job("c")) + _, ackR := pop(t, q) + _, ackC := pop(t, q) + if !q.AcknowledgeResult(ackR, []byte(`"ok"`)) || !q.AcknowledgeCancelled(ackC, "stop") { + t.Fatal("finish failed") + } + if countRows(t, q.f.db, "status = 'completed'") != 1 || countRows(t, q.f.db, "status = 'cancelled'") != 1 { + t.Error("Expected one completed and one cancelled row") + } + if got := waitOutcome(t, watch(q, "r")); string(got.Result) != `"ok"` { + t.Errorf("got %+v", got) + } + if got := waitOutcome(t, watch(q, "c")); got.State != contract.JobCancelled || got.Error != "stop" { + t.Errorf("got %+v", got) + } +} + +// A read reaches only watches started before it: an earlier run's record, +// read before a re-add, never ends a watch made after. +func TestDeliverSkipsNewerWatches(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + ch := watch(q, "x") + q.w.mu.Lock() + gen := q.w.gen + q.w.mu.Unlock() + q.deliver(contract.JobOutcome{ID: "x", State: contract.JobCompleted}, gen-1) + noOutcome(t, ch) + q.deliver(contract.JobOutcome{ID: "x", State: contract.JobCompleted}, gen) + waitOutcome(t, ch) +} + +// Every watch fires when far more jobs end between two polls than +// retention keeps, and more ids are watched than one read used to take. +func TestWatchBurst(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q2 := otherProcess(t, q) + const n = 3 * keepCompleted + var fired atomic.Int64 + done := make(chan struct{}) + for i := range n { + id := "b" + strconv.Itoa(i) + q.WatchOutcome(id, func(contract.JobOutcome) { + if fired.Add(1) == n { + close(done) + } + }) + q.Enqueue(job(id)) + } + items, acks := make([]any, 500), make([]string, 500) + for { + k, _, err := q2.DequeueBatchWithAckIds(items, acks) + if err != nil || k == 0 { + break + } + for _, a := range acks[:k] { + q2.Acknowledge(a) + } + } + select { + case <-done: + case <-time.After(20 * time.Second): + t.Fatalf("%d of %d watches fired", fired.Load(), n) + } +} + +// Queue names may not hold __sqliteq_, which names a queue's own tables. +func TestReservedQueueName(t *testing.T) { + qs := mustNew(t, filepath.Join(t.TempDir(), "one.db"), WithSingleFile()) + defer qs.Close() + if _, err := qs.NewQueue("orders__sqliteq_jobs"); err == nil { + t.Error("Expected a reserved name to fail") + } +} From 665759faee32989f76ea2ba22a075cf8521483a0 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 22:21:03 +0600 Subject: [PATCH 08/10] test: sweep, single pops past the drop cap Codecov: sweep was uncovered; DequeueWithAckId and Dequeue loop again past maxDrops, now tested. --- queue_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/queue_test.go b/queue_test.go index c12eb7d..17b975d 100644 --- a/queue_test.go +++ b/queue_test.go @@ -1397,6 +1397,42 @@ func TestCancelDropCap(t *testing.T) { if n := countJobs(t, q); n != maxDrops { t.Errorf("Expected %d cancelled records, got %d", maxDrops, n) } + + // Single-item pops can't report items left: past the cap they pop again + for _, p := range []struct { + name string + pop func() (any, bool) + }{ + {"DequeueWithAckId", func() (any, bool) { v, ok, _ := q.DequeueWithAckId(); return v, ok }}, + {"Dequeue", q.Dequeue}, + } { + for i := range maxDrops + 1 { + id := p.name + strconv.Itoa(i) + q.Enqueue(job(id)) + q.CancelJob(id) + } + q.Enqueue([]byte("plain")) + if v, ok := p.pop(); !ok || string(v.([]byte)) != "plain" { + t.Errorf("%s past the cap: got %v %v, want plain", p.name, v, ok) + } + } +} + +// sweep trims records past their age and drops expired marks. +func TestSweep(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + old := time.Now().Add(-keepFailedAge).UnixMilli() + q.f.db.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", old) + q.f.db.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'failed', ?)", time.Now().UnixMilli()) + q.f.db.Exec("INSERT INTO "+q.cancelled+" (job_id, expires_at) VALUES ('gone', ?)", old) + q.CancelJob("live") + q.sweep() + var marks int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks) + if countJobs(t, q) != 1 || marks != 1 { + t.Errorf("Expected 1 record and 1 mark left, got %d and %d", countJobs(t, q), marks) + } } // Queues.Close stops polls and drops live watches and hooks without From f42847fb0802d1cf8ae2dab067cfe53f1b3858cc Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 22:22:20 +0600 Subject: [PATCH 09/10] build: pin contract v0.6.0, varmq main 6bc553d varmq#179 merged; contract tagged. --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 6b04aeb..d59e374 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/goptics/sqliteq go 1.27 require ( - github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092 - github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092 + github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6 + github.com/goptics/varmq/contract v0.6.0 github.com/mattn/go-sqlite3 v1.14.28 ) diff --git a/go.sum b/go.sum index f4edda0..f800d0f 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092 h1:oFPxzb1mXotIzHHKlxz+kUGO1ASrWZkg2nmkYhrgx60= -github.com/goptics/varmq v1.10.1-0.20261108145618-0a9288ee1092/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E= -github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092 h1:wWq1t3n0//1/URO+uwwdEybf6GJcVHmmtJ9lcKAHxRY= -github.com/goptics/varmq/contract v0.4.1-0.20261108145618-0a9288ee1092/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg= +github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6 h1:0GooGJC22Z/9WxAz0f8aVzWyPkBDJhHF8JI4oYTIW+w= +github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E= +github.com/goptics/varmq/contract v0.6.0 h1://rpDlzSwsqvPDS98siuPDN1LyyBLFhiYeAmB7MZfbQ= +github.com/goptics/varmq/contract v0.6.0/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From e15946977ed7a9f7ae751cdf5fa3bdf0c983b35b Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Sun, 8 Nov 2026 22:22:20 +0600 Subject: [PATCH 10/10] test: undecodable envelope is a plain job Bad JSON, non-string or empty id: handed out by a gated pop, finishes as usual, no record. --- queue_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/queue_test.go b/queue_test.go index 17b975d..6f55614 100644 --- a/queue_test.go +++ b/queue_test.go @@ -1650,3 +1650,30 @@ func TestReservedQueueName(t *testing.T) { t.Error("Expected a reserved name to fail") } } + +// An item whose envelope does not decode to a string id is a plain job: +// with a live mark (gated pops) it is handed out, finishes as usual, and +// writes no record. +func TestUndecodableEnvelopeIsPlainJob(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.CancelJob("other") // keeps the pops gated + bad := []string{`{bad`, `{"id":5}`, `{"id":null}`, `{"id":""}`, `[1]`, "plain", `{"id":"x"`} + for _, b := range bad { + q.Enqueue([]byte(b)) + } + for i, b := range bad { + v, ack := pop(t, q) + if v != b { + t.Fatalf("pop %d: got %q, want %q", i, v, b) + } + if i%2 == 0 { + q.Acknowledge(ack) + } else { + q.Reject(ack, "r") + } + } + if n := countJobs(t, q); n != 0 { + t.Errorf("Expected no records, got %d", n) + } +}