diff --git a/.github/workflows/sqliteq.yml b/.github/workflows/sqliteq.yml
index b658b4d..bf9d168 100644
--- a/.github/workflows/sqliteq.yml
+++ b/.github/workflows/sqliteq.yml
@@ -24,7 +24,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
- go-version: "1.24.1"
+ go-version: "1.27.0"
cache: true
- name: Install dependencies
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2b5eabe..8961d18 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,7 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
-- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is not stored.
+- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error.
+- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it by seq every 50 ms (new records only), from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d; the count limit spares records under 1 min old. A polling queue object also sweeps retention and expired marks once a minute.
+- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per transaction; a single-item pop goes on past that, a batch pop reports an item left); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended.
+- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks.
+
+### Breaking
+
+- Requires Go 1.27 (was 1.24), for the varmq dependency.
+- A queue name must not contain `__sqliteq_`, in both layouts: a queue's indexes, sidecar tables and trigger use it.
+
+### Changed
+
+- Transactions begin `IMMEDIATE`. A finish (`Acknowledge`, `Reject`) always runs in a transaction; a pop runs in one while any cancel mark is live.
## [0.3.0] - 2026-10-23
diff --git a/README.md b/README.md
index d911546..6058b8b 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@
-
+
@@ -49,7 +49,7 @@
go get github.com/goptics/sqliteq
```
-Requires Go 1.24+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler.
+Requires Go 1.27+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler.
## Quick Start
diff --git a/cancel.go b/cancel.go
new file mode 100644
index 0000000..6b4445e
--- /dev/null
+++ b/cancel.go
@@ -0,0 +1,192 @@
+package sqliteq
+
+import (
+ "database/sql"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/goptics/varmq/contract"
+)
+
+// Job cancel (varmq spec/redis-storage.md 1.7.0, 6.14), on SQLite.
+// CancelJob writes a mark to __sqliteq_cancelled (expires_at: ms).
+// Every pop drops a marked waiting job with outcome cancelled; OnCancel
+// learns of marks by polling seq; AcknowledgeCancelled finishes a running
+// job that stopped for it. Any finish, and an EnqueueOnce that stores the
+// id, clears the mark. sqliteq has no workflow store, so no item is exempt.
+
+var (
+ _ contract.JobCanceller = (*Queue)(nil)
+ _ contract.JobCanceller = (*PriorityQueue)(nil)
+ _ contract.CancelNotifier = (*Queue)(nil)
+ _ contract.CancelNotifier = (*PriorityQueue)(nil)
+ _ contract.CancelAcknowledgeable = (*Queue)(nil)
+ _ contract.CancelAcknowledgeable = (*PriorityQueue)(nil)
+)
+
+const (
+ // cancelTTL is how long one mark lasts.
+ cancelTTL = 24 * time.Hour
+ // maxDrops caps the marked items one pop drops; at the cap it returns
+ // what it has and the next pop goes on.
+ maxDrops = 1000
+)
+
+var (
+ errEmptyJobID = errors.New("sqliteq: empty job id")
+ errClosed = errors.New("sqliteq: queue is closed")
+)
+
+// CancelJob implements contract.JobCanceller: it marks job id cancelled for
+// cancelTTL. A waiting job is dropped by the pop that would hand it out
+// (outcome cancelled); OnCancel callbacks of every queue object on the file
+// learn of a running one, which ends only if its worker stops it.
+// contract.ErrJobEnded when its outcome record shows it ended. An unknown
+// id is marked too: a later Enqueue of it is dropped, unless EnqueueOnce
+// stores it (that clears the mark).
+func (q *Queue) CancelJob(id string) error {
+ if id == "" {
+ return errEmptyJobID
+ }
+ if q.closed.Load() {
+ return errClosed
+ }
+
+ now := time.Now().UnixMilli()
+ q.f.mu.Lock()
+ defer q.f.mu.Unlock()
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return err
+ }
+ defer tx.Rollback()
+
+ var state string
+ switch err := tx.QueryRow(fmt.Sprintf("SELECT state FROM %s WHERE job_id = ?", q.jobs), id).Scan(&state); {
+ case err == nil:
+ return contract.ErrJobEnded // every record is an end
+ case !errors.Is(err, sql.ErrNoRows):
+ return err
+ }
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE expires_at <= ?", q.cancelled), now); err != nil {
+ return err
+ }
+ // REPLACE deletes an older mark of id, so the new one takes a new seq
+ // and reaches OnCancel again
+ if _, err := tx.Exec(fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, expires_at) VALUES (?, ?)", q.cancelled),
+ id, now+cancelTTL.Milliseconds()); err != nil {
+ return err
+ }
+ return tx.Commit()
+}
+
+// AcknowledgeCancelled implements contract.CancelAcknowledgeable: Reject
+// with outcome cancelled and reason as its error. A kept row
+// (WithRemoveOnComplete(false)) gets status 'cancelled'.
+func (q *Queue) AcknowledgeCancelled(ackID, reason string) bool {
+ return q.finish(ackID, contract.JobCancelled, reason, nil)
+}
+
+// OnCancel implements contract.CancelNotifier: fn gets the job ID of each
+// cancel request made after the call, through any queue object on this
+// database file, until stop. Marks are polled every pollEvery; each call
+// runs on its own goroutine. A mark gone by the poll (the job finished or
+// was re-added) is never reported. A closed queue, or a failed first read,
+// reports nothing.
+func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) {
+ // Read before the lock: a mark made after this read has a higher seq
+ var last sql.NullInt64
+ if err := q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.cancelled)).Scan(&last); err != nil {
+ return func() {} // 0 would replay every live mark
+ }
+
+ h := &cancelHook{fn: fn, from: last.Int64}
+ w := &q.w
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.closed {
+ return func() {}
+ }
+ if len(w.hooks) == 0 {
+ w.lastSeq = h.from // no poll read marks for anyone since
+ }
+ if w.hooks == nil {
+ w.hooks = map[*cancelHook]struct{}{}
+ }
+ w.hooks[h] = struct{}{}
+ w.startLocked(q)
+
+ return func() {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ delete(w.hooks, h)
+ }
+}
+
+// cancelHook is one OnCancel callback; it gets marks with seq above from.
+type cancelHook struct {
+ fn func(string)
+ from int64
+}
+
+// pollCancels reads the live marks newer than seq and passes each to the
+// hooks live after the read whose from is below it, so a hook added during
+// the read still gets its marks, and a stopped one gets none.
+func (q *Queue) pollCancels(seq int64) {
+ rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? AND expires_at > ? ORDER BY seq LIMIT %d",
+ q.cancelled, maxDrops), seq, time.Now().UnixMilli())
+ if err != nil {
+ return
+ }
+ type mark struct {
+ seq int64
+ id string
+ }
+ var marks []mark
+ for rows.Next() {
+ var m mark
+ if rows.Scan(&m.seq, &m.id) == nil {
+ marks = append(marks, m)
+ }
+ }
+ if rows.Close() != nil || rows.Err() != nil || len(marks) == 0 {
+ return // read again from the old seq
+ }
+
+ w := &q.w
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.closed {
+ return
+ }
+ w.lastSeq = max(w.lastSeq, marks[len(marks)-1].seq)
+ for h := range w.hooks {
+ for _, m := range marks {
+ if m.seq > h.from {
+ go h.fn(m.id)
+ }
+ }
+ }
+}
+
+// anyMark reports whether a live mark exists, so a pop without marks pays
+// one read; an error counts as a mark (the gated pop then reports it).
+func (q *Queue) anyMark(now time.Time) bool {
+ var n int
+ if err := q.hasMark.QueryRow(now.UnixMilli()).Scan(&n); err != nil {
+ return true
+ }
+ return n == 1
+}
+
+// marked reports whether job id has a live mark; isMarked is q.isMarked
+// in the caller's tx.
+func marked(isMarked *sql.Stmt, id string, now time.Time) (bool, error) {
+ if id == "" {
+ return false, nil
+ }
+ var n int
+ err := isMarked.QueryRow(id, now.UnixMilli()).Scan(&n)
+ return n == 1, err
+}
diff --git a/commit.go b/commit.go
index edb4f29..be6fed2 100644
--- a/commit.go
+++ b/commit.go
@@ -10,7 +10,7 @@ import (
// code (such as a driver.Valuer): a panic there would strand the round and
// leave its transaction open. Convert user values before building it.
type op struct {
- run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit
+ run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit, or a tx of its own
done chan result // buffered 2: one lead wake-up and the outcome
}
@@ -97,6 +97,7 @@ func (b *batcher) lead() {
// transaction of its own, so one bad call fails alone. A round of one op runs
// with a nil tx, as one autocommit statement: a database/sql Tx costs a
// context and a goroutine, which a round with nothing to share does not repay.
+// An op of several statements (a finish) opens its own tx then.
func (f *file) commitRound(ops []*op) {
f.mu.Lock()
defer f.mu.Unlock()
diff --git a/commit_test.go b/commit_test.go
index 6b1a7e1..54fa30d 100644
--- a/commit_test.go
+++ b/commit_test.go
@@ -491,3 +491,53 @@ func TestGroupCommitAfterClose(t *testing.T) {
}
waitAll(t, &wg, 10*time.Second)
}
+
+// Finishes that share a round write their outcomes in its transaction,
+// and again each in its own when a bad call fails the round.
+func TestGroupFinishOutcomes(t *testing.T) {
+ for _, fail := range []bool{false, true} {
+ t.Run(map[bool]string{false: "shared", true: "fallback"}[fail], func(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ for _, id := range []string{"a", "b", "c", "d"} {
+ q.Enqueue(job(id))
+ }
+ items, acks := make([]any, 4), make([]string, 4)
+ if n, _, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != 4 {
+ t.Fatalf("got %d %v", n, err)
+ }
+ rest := []func() bool{
+ func() bool { return q.Reject(acks[1], "r") },
+ func() bool { return q.AcknowledgeResult(acks[2], []byte(`1`)) },
+ func() bool { return q.AcknowledgeCancelled(acks[3], "c") },
+ }
+ if fail {
+ rest = append(rest, func() bool { return q.Enqueue(nil) })
+ }
+ got := oneRound(t, q.f, func() bool { return q.Acknowledge(acks[0]) }, rest)
+ for i, ok := range got[:4] {
+ if !ok {
+ t.Errorf("finish %d failed", i)
+ }
+ }
+ want := map[string]string{"a": "completed||", "b": "failed|r|", "c": "completed||1", "d": "cancelled|c|"}
+ rows, err := q.f.db.Query("SELECT job_id, state, COALESCE(error, ''), COALESCE(CAST(result AS TEXT), '') FROM " + q.jobs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer rows.Close()
+ n := 0
+ for rows.Next() {
+ var id, state, e, r string
+ rows.Scan(&id, &state, &e, &r)
+ if g := state + "|" + e + "|" + r; g != want[id] {
+ t.Errorf("%s: got %s, want %s", id, g, want[id])
+ }
+ n++
+ }
+ if n != 4 {
+ t.Errorf("Expected 4 records, got %d", n)
+ }
+ })
+ }
+}
diff --git a/go.mod b/go.mod
index b6ff707..d59e374 100644
--- a/go.mod
+++ b/go.mod
@@ -1,5 +1,9 @@
module github.com/goptics/sqliteq
-go 1.24
+go 1.27
-require github.com/mattn/go-sqlite3 v1.14.28
+require (
+ github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6
+ github.com/goptics/varmq/contract v0.6.0
+ github.com/mattn/go-sqlite3 v1.14.28
+)
diff --git a/go.sum b/go.sum
index 42e5bac..f800d0f 100644
--- a/go.sum
+++ b/go.sum
@@ -1,2 +1,14 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6 h1:0GooGJC22Z/9WxAz0f8aVzWyPkBDJhHF8JI4oYTIW+w=
+github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E=
+github.com/goptics/varmq/contract v0.6.0 h1://rpDlzSwsqvPDS98siuPDN1LyyBLFhiYeAmB7MZfbQ=
+github.com/goptics/varmq/contract v0.6.0/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg=
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/migration_test.go b/migration_test.go
index bc94204..be706f8 100644
--- a/migration_test.go
+++ b/migration_test.go
@@ -129,6 +129,12 @@ func TestMigrateV02File(t *testing.T) {
if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) {
t.Errorf("Round %d: indexes = %v, want %v", round, got, want)
}
+ for _, side := range []string{name + jobsSuffix, name + cancelledSuffix, name + readdSuffix} {
+ var n int
+ if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = ?", side).Scan(&n); err != nil || n != 1 {
+ t.Errorf("Round %d: expected %s on the old file, got %d %v", round, side, n, err)
+ }
+ }
if round == 1 {
// The two processing rows are pending again after recovery
diff --git a/outcome.go b/outcome.go
new file mode 100644
index 0000000..643f2a3
--- /dev/null
+++ b/outcome.go
@@ -0,0 +1,424 @@
+package sqliteq
+
+import (
+ "database/sql"
+ "encoding/json"
+ "fmt"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/goptics/varmq/contract"
+)
+
+// Job outcome (varmq spec/redis-storage.md 1.6.0, 6.13), on SQLite. Each
+// finish of a job with a Job ID writes one row to __sqliteq_jobs in
+// the finish's transaction, under a new AUTOINCREMENT seq. SQLite has no
+// pub/sub, so a watcher is answered by polling that table by seq (records
+// newer than the last read): one goroutine per queue object, alive only
+// while it has watches or OnCancel hooks. It also sweeps retention and
+// expired marks.
+
+var (
+ _ contract.OutcomeWatcher = (*Queue)(nil)
+ _ contract.OutcomeWatcher = (*PriorityQueue)(nil)
+ _ contract.ResultAcknowledgeable = (*Queue)(nil)
+ _ contract.ResultAcknowledgeable = (*PriorityQueue)(nil)
+)
+
+const (
+ // pollEvery is how often a queue object with watches or hooks reads
+ // the outcome and cancel tables.
+ pollEvery = 50 * time.Millisecond
+ // pollBatch caps the rows one cursor read returns; a poll reads again
+ // until it gets fewer.
+ pollBatch = 1000
+ // sweepEvery is how often a polling queue object trims retention and
+ // drops expired marks.
+ sweepEvery = time.Minute
+
+ // Retention (spec 5 defaults): cancelled records count as failed.
+ keepCompleted = 1000
+ keepCompletedAge = 24 * time.Hour
+ keepFailed = 10000
+ keepFailedAge = 7 * 24 * time.Hour
+ // trimEvery is how many records a queue object writes between trims.
+ trimEvery = 100
+ // trimFloor: the count limit never drops a record younger than this,
+ // so a burst can't trim one before a poll reads it.
+ trimFloor = time.Minute
+
+ jobsSuffix = "__sqliteq_jobs"
+ cancelledSuffix = "__sqliteq_cancelled"
+ readdSuffix = "__sqliteq_readd"
+)
+
+// initSidecars creates the outcome and cancel tables, and the trigger that
+// clears both for an id EnqueueOnce stores (spec 6.13 and 6.14 Re-add): an
+// ON CONFLICT DO NOTHING insert of a repeat id fires no trigger. The caller
+// holds the write mutex.
+func (q *Queue) initSidecars() error {
+ for _, s := range []string{
+ // AUTOINCREMENT: seq never reuses a deleted row's, so a seq > last
+ // cursor sees every new record and mark; REPLACE gives a new seq
+ `CREATE TABLE IF NOT EXISTS %[2]s (
+ seq INTEGER PRIMARY KEY AUTOINCREMENT,
+ job_id TEXT NOT NULL UNIQUE,
+ state TEXT NOT NULL,
+ error TEXT,
+ result BLOB,
+ finished_at INTEGER NOT NULL
+ )`,
+ `CREATE INDEX IF NOT EXISTS %[4]s ON %[2]s (finished_at)`,
+ `CREATE TABLE IF NOT EXISTS %[3]s (
+ seq INTEGER PRIMARY KEY AUTOINCREMENT,
+ job_id TEXT NOT NULL UNIQUE,
+ expires_at INTEGER NOT NULL
+ )`,
+ `CREATE INDEX IF NOT EXISTS %[6]s ON %[3]s (expires_at)`,
+ `CREATE TRIGGER IF NOT EXISTS %[5]s AFTER INSERT ON %[1]s WHEN NEW.job_id IS NOT NULL BEGIN
+ DELETE FROM %[2]s WHERE job_id = NEW.job_id;
+ DELETE FROM %[3]s WHERE job_id = NEW.job_id;
+ END`,
+ } {
+ query := fmt.Sprintf(s, q.table, q.jobs, q.cancelled,
+ quoteIdent(q.name+jobsSuffix+"_finished"), quoteIdent(q.name+readdSuffix),
+ quoteIdent(q.name+cancelledSuffix+"_expires"))
+ if _, err := q.f.db.Exec(query); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// jobIDOf is a row's Job ID: its EnqueueOnce id, else the envelope "id" of
+// a JSON object item (varmq's generated ids come with plain Enqueue). ""
+// when it has neither: such a job has no outcome and is never cancelled.
+func jobIDOf(onceID sql.NullString, data []byte) string {
+ if onceID.Valid {
+ return onceID.String
+ }
+ if len(data) == 0 || data[0] != '{' {
+ return ""
+ }
+ var env struct {
+ ID string `json:"id"`
+ }
+ if json.Unmarshal(data, &env) != nil {
+ return ""
+ }
+ return env.ID
+}
+
+// record writes job id's outcome and drops its cancel mark, in tx; every
+// trimEvery-th record also trims both states.
+func (q *Queue) record(tx *sql.Tx, id string, state contract.JobState, reason string, result []byte, now time.Time) error {
+ var errText any
+ if state != contract.JobCompleted {
+ errText = reason
+ }
+ if _, err := tx.Stmt(q.putJob).Exec(id, string(state), errText, result, now.UnixMilli()); err != nil {
+ return err
+ }
+ if _, err := tx.Stmt(q.unmark).Exec(id); err != nil {
+ return err
+ }
+ // The first record of a queue object trims too: a short-lived process
+ // may write fewer than trimEvery
+ if q.records.Add(1)%trimEvery == 1 {
+ return q.trim(tx, now)
+ }
+ return nil
+}
+
+// trim applies retention: per state group, records past the age limit and
+// those past the newest N that are older than trimFloor.
+func (q *Queue) trim(tx *sql.Tx, now time.Time) error {
+ for _, g := range []struct {
+ where string
+ keep int
+ age time.Duration
+ }{
+ {"state = 'completed'", keepCompleted, keepCompletedAge},
+ {"state <> 'completed'", keepFailed, keepFailedAge},
+ } {
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE %s AND finished_at <= ?", q.jobs, g.where),
+ now.Add(-g.age).UnixMilli()); err != nil {
+ return err
+ }
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %[1]s WHERE finished_at <= ? AND job_id IN (SELECT job_id FROM %[1]s WHERE %[2]s ORDER BY finished_at DESC LIMIT -1 OFFSET %[3]d)",
+ q.jobs, g.where, g.keep), now.Add(-trimFloor).UnixMilli()); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// AcknowledgeResult implements contract.ResultAcknowledgeable: Acknowledge
+// that keeps result, which must be valid JSON, in the job's outcome. Not
+// JSON: false, and nothing changes.
+func (q *Queue) AcknowledgeResult(ackID string, result []byte) bool {
+ if !json.Valid(result) {
+ return false
+ }
+ return q.finish(ackID, contract.JobCompleted, "", result)
+}
+
+type outcomeWatch struct {
+ fn func(contract.JobOutcome)
+ gen uint64 // w.gen at its start
+ done atomic.Bool // called or cancelled
+}
+
+// watchers are a queue object's outcome watches and OnCancel hooks, and the
+// state of its one poll goroutine. mu is never held across a query.
+type watchers struct {
+ mu sync.Mutex
+ byID map[string]map[*outcomeWatch]struct{}
+ hooks map[*cancelHook]struct{}
+ lastSeq int64 // newest cancel mark seq a poll has read
+ outSeq int64 // newest outcome record seq a poll has read
+ gen uint64 // watches started; a read reaches only watches started before it
+ polling bool
+ closed bool
+ stop chan struct{} // closed by close
+}
+
+// startLocked starts the poll goroutine if none runs; w.mu is held.
+func (w *watchers) startLocked(q *Queue) {
+ if !w.polling {
+ w.polling = true
+ go q.poll()
+ }
+}
+
+// close drops every watch and hook without calling it, and stops polling.
+func (w *watchers) close() {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.closed {
+ return
+ }
+ w.closed = true
+ close(w.stop)
+ for _, set := range w.byID {
+ for o := range set {
+ o.done.Store(true)
+ }
+ }
+ w.byID, w.hooks = nil, nil
+}
+
+// WatchOutcome implements contract.OutcomeWatcher: fn runs once, on its own
+// goroutine, when job id completes, fails or is cancelled through any
+// queue object on this database file, in this process or another. A job
+// whose record is still kept is reported even if it ended before the call.
+// It is polled every pollEvery. An empty id, or a closed queue, watches
+// nothing. Close drops watches without calling fn.
+func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel func()) {
+ o := &outcomeWatch{fn: fn}
+ w := &q.w
+ w.mu.Lock()
+ if id == "" || w.closed {
+ w.mu.Unlock()
+ return func() {}
+ }
+ if len(w.byID) == 0 {
+ // No poll moved the cursor while nothing was watched: start it at
+ // the newest record. Older ones are this watch's catchUp below.
+ w.mu.Unlock()
+ var last sql.NullInt64
+ err := q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.jobs)).Scan(&last)
+ w.mu.Lock()
+ if w.closed {
+ w.mu.Unlock()
+ return func() {}
+ }
+ if err == nil && len(w.byID) == 0 {
+ w.outSeq = last.Int64
+ }
+ }
+ w.gen++
+ o.gen = w.gen
+ if w.byID == nil {
+ w.byID = map[string]map[*outcomeWatch]struct{}{}
+ }
+ if w.byID[id] == nil {
+ w.byID[id] = map[*outcomeWatch]struct{}{}
+ }
+ w.byID[id][o] = struct{}{}
+ w.startLocked(q)
+ w.mu.Unlock()
+
+ go q.catchUp([]string{id}, o.gen)
+
+ return func() {
+ if o.done.CompareAndSwap(false, true) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if set := w.byID[id]; set != nil {
+ delete(set, o)
+ if len(set) == 0 {
+ delete(w.byID, id)
+ }
+ }
+ }
+ }
+}
+
+// poll reads new outcome records and cancel marks every pollEvery, and
+// sweeps every sweepEvery, until Close or until nothing is watched.
+func (q *Queue) poll() {
+ t := time.NewTicker(pollEvery)
+ defer t.Stop()
+ w := &q.w
+ swept := time.Now()
+ for {
+ select {
+ case <-w.stop:
+ return
+ case <-t.C:
+ }
+
+ w.mu.Lock()
+ if len(w.byID) == 0 && len(w.hooks) == 0 {
+ w.polling = false
+ w.mu.Unlock()
+ return
+ }
+ watching, hooks := len(w.byID) > 0, len(w.hooks) > 0
+ outSeq, seq, gen := w.outSeq, w.lastSeq, w.gen
+ w.mu.Unlock()
+
+ if watching {
+ q.pollOutcomes(outSeq, gen)
+ }
+ if hooks {
+ q.pollCancels(seq)
+ }
+ if time.Since(swept) >= sweepEvery {
+ swept = time.Now()
+ q.sweep()
+ }
+ }
+}
+
+// pollOutcomes reads the records newer than seq, in pollBatch reads, and
+// delivers the watched ones to watches up to gen.
+func (q *Queue) pollOutcomes(seq int64, gen uint64) {
+ for {
+ rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? ORDER BY seq LIMIT %d", q.jobs, pollBatch), seq)
+ if err != nil {
+ return
+ }
+ var ids []string
+ for rows.Next() {
+ var id string
+ if rows.Scan(&seq, &id) == nil {
+ ids = append(ids, id)
+ }
+ }
+ if rows.Close() != nil || rows.Err() != nil || len(ids) == 0 {
+ return // read again from the old seq
+ }
+
+ w := &q.w
+ w.mu.Lock()
+ w.outSeq = max(w.outSeq, seq)
+ watched := ids[:0]
+ for _, id := range ids {
+ if w.byID[id] != nil {
+ watched = append(watched, id)
+ }
+ }
+ w.mu.Unlock()
+ if len(watched) > 0 {
+ q.catchUp(watched, gen)
+ }
+ if len(ids) < pollBatch {
+ return
+ }
+ }
+}
+
+// sweep applies retention and drops expired cancel marks (spec 5, 6.14
+// Prune), so an idle queue ages out too.
+func (q *Queue) sweep() {
+ q.f.mu.Lock()
+ defer q.f.mu.Unlock()
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return
+ }
+ defer tx.Rollback()
+ now := time.Now()
+ if q.trim(tx, now) != nil {
+ return
+ }
+ if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE expires_at <= ?", q.cancelled), now.UnixMilli()); err != nil {
+ return
+ }
+ tx.Commit()
+}
+
+// catchUp delivers the ids whose outcome record exists to their watches
+// up to gen, those started before the read. A failed read is retried by
+// the next poll.
+func (q *Queue) catchUp(ids []string, gen uint64) {
+ args := make([]any, len(ids))
+ for i, id := range ids {
+ args[i] = id
+ }
+ rows, err := q.f.db.Query(fmt.Sprintf("SELECT job_id, state, error, result FROM %s WHERE job_id IN (%s)",
+ q.jobs, strings.Repeat("?, ", len(ids)-1)+"?"), args...)
+ if err != nil {
+ return
+ }
+ var outs []contract.JobOutcome
+ for rows.Next() {
+ var (
+ id, state string
+ errText sql.NullString
+ result []byte
+ )
+ if rows.Scan(&id, &state, &errText, &result) != nil {
+ continue
+ }
+ outs = append(outs, contract.JobOutcome{ID: id, State: contract.JobState(state), Error: errText.String, Result: result})
+ }
+ rows.Close()
+
+ for _, out := range outs {
+ q.deliver(out, gen)
+ }
+}
+
+// deliver ends the watches of out.ID started up to gen: each fn runs once,
+// on its own goroutine, so a slow fn holds up no poll. A watch started
+// after the read stays: the record read may be an earlier run's, which a
+// re-add since deleted (spec 6.13 Re-add).
+func (q *Queue) deliver(out contract.JobOutcome, gen uint64) {
+ w := &q.w
+ w.mu.Lock()
+ set := w.byID[out.ID]
+ var ended []*outcomeWatch
+ for o := range set {
+ if o.gen <= gen {
+ ended = append(ended, o)
+ delete(set, o)
+ }
+ }
+ if len(set) == 0 {
+ delete(w.byID, out.ID)
+ }
+ w.mu.Unlock()
+
+ for _, o := range ended {
+ go func() {
+ if o.done.CompareAndSwap(false, true) {
+ o.fn(out)
+ }
+ }()
+ }
+}
diff --git a/priority_queue_test.go b/priority_queue_test.go
index 39b2ed7..a7cadcf 100644
--- a/priority_queue_test.go
+++ b/priority_queue_test.go
@@ -1,10 +1,13 @@
package sqliteq
import (
+ "errors"
"fmt"
"strconv"
"testing"
+ "github.com/goptics/varmq/contract"
+
_ "github.com/mattn/go-sqlite3"
)
@@ -653,3 +656,44 @@ func TestBatchPriorityOrder(t *testing.T) {
}
}
}
+
+// A priority queue reports outcomes, drops marked waiting jobs in priority
+// order, and re-add clears the mark, as Queue does.
+func TestPriorityOutcomeAndCancel(t *testing.T) {
+ qs := mustNew(t, t.TempDir())
+ defer qs.Close()
+ pq, err := qs.NewPriorityQueue("pq")
+ if err != nil {
+ t.Fatal(err)
+ }
+ other := otherProcess(t, pq.Queue)
+
+ pq.Enqueue(job("low"), 5)
+ pq.Enqueue(job("high"), 1)
+ pq.EnqueueOnce(job("mid"), 3, "mid")
+ if err := other.CancelJob("high"); err != nil {
+ t.Fatal(err)
+ }
+ chHigh, chMid := watch(other, "high"), watch(other, "mid")
+
+ if v, ack := pop(t, pq.Queue); v != string(job("mid")) {
+ t.Fatalf("got %s, want mid", v)
+ } else if !pq.AcknowledgeResult(ack, []byte(`7`)) {
+ t.Fatal("AcknowledgeResult failed")
+ }
+ if got := waitOutcome(t, chHigh); got.State != contract.JobCancelled {
+ t.Errorf("high: got %+v", got)
+ }
+ if got := waitOutcome(t, chMid); got.State != contract.JobCompleted || string(got.Result) != "7" {
+ t.Errorf("mid: got %+v", got)
+ }
+ if err := pq.CancelJob("mid"); !errors.Is(err, contract.ErrJobEnded) {
+ t.Errorf("got %v, want ErrJobEnded", err)
+ }
+
+ pq.CancelJob("again")
+ pq.EnqueueOnce(job("again"), 0, "again")
+ if v, _ := pop(t, pq.Queue); v != string(job("again")) {
+ t.Errorf("Expected the re-added job first, got %s", v)
+ }
+}
diff --git a/queue.go b/queue.go
index 7d41be6..be1fbdd 100644
--- a/queue.go
+++ b/queue.go
@@ -4,6 +4,7 @@ import (
"cmp"
"database/sql"
"database/sql/driver"
+ "errors"
"fmt"
"slices"
"strconv"
@@ -11,6 +12,7 @@ import (
"sync/atomic"
"time"
+ "github.com/goptics/varmq/contract"
_ "github.com/mattn/go-sqlite3"
)
@@ -26,8 +28,13 @@ type Queue struct {
removeOnComplete bool
closed atomic.Bool
+ jobs, cancelled string // quoted outcome and cancel mark tables
+ w watchers // outcome watches and OnCancel hooks
+ records atomic.Int64 // outcome records written, for trims
+
// Prepared once per handle; they close with the file's pool
- insert, claim, pop, ack, reject, count, countCapped, values *sql.Stmt
+ insert, claim, pop, fin, count, countCapped, values *sql.Stmt
+ putJob, unmark, hasMark, isMarked *sql.Stmt
}
// claimed is one row a claim moved to processing
@@ -35,6 +42,7 @@ type claimed struct {
id int64
priority int
data []byte
+ once sql.NullString // job_id
}
// newQueue creates a SQLite-based queue on its table: the table itself, the
@@ -48,7 +56,10 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro
priority: priority,
order: "id",
removeOnComplete: true, // Default to removing completed items
+ jobs: quoteIdent(name + jobsSuffix),
+ cancelled: quoteIdent(name + cancelledSuffix),
}
+ q.w.stop = make(chan struct{})
if priority {
q.order = "priority, id"
}
@@ -86,6 +97,9 @@ func (q *Queue) initSchema() error {
if err := q.addJobID(); err != nil {
return fmt.Errorf("failed to add job_id: %w", err)
}
+ if err := q.initSidecars(); err != nil {
+ return fmt.Errorf("failed to create outcome tables: %w", err)
+ }
return nil
}
@@ -198,18 +212,16 @@ func (q *Queue) prepare() (err error) {
}
}()
- cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data"
+ cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data, job_id"
if q.priority {
- cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data"
+ cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data, job_id"
}
pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)
- ack := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing'", q.table)
- reject := ack
+ // fin ends a processing row: deleted, or kept with its end as status
+ fin := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table)
if !q.removeOnComplete {
- finish := "UPDATE %s SET status = '%s', ack = 1, updated_at = ? WHERE id = ? AND status = 'processing'"
- ack = fmt.Sprintf(finish, q.table, "completed")
- reject = fmt.Sprintf(finish, q.table, "failed")
+ fin = fmt.Sprintf("UPDATE %s SET status = ?, ack = 1, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table)
}
for _, s := range []struct {
@@ -220,9 +232,12 @@ func (q *Queue) prepare() (err error) {
// other failure still errors; it must repeat the index's WHERE
{&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING", q.table, cols, vals)},
{&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)},
- {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)},
- {&q.ack, ack},
- {&q.reject, reject},
+ {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data, job_id", q.table, pending)},
+ {&q.fin, fin},
+ {&q.putJob, fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, state, error, result, finished_at) VALUES (?, ?, ?, ?, ?)", q.jobs)},
+ {&q.unmark, fmt.Sprintf("DELETE FROM %s WHERE job_id = ?", q.cancelled)},
+ {&q.hasMark, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE expires_at > ?)", q.cancelled)},
+ {&q.isMarked, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE job_id = ? AND expires_at > ?)", q.cancelled)},
{&q.count, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", q.table)},
{&q.countCapped, fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE status = 'pending' LIMIT %d)", q.table, remainingCap)},
{&q.values, fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)},
@@ -237,7 +252,7 @@ func (q *Queue) prepare() (err error) {
// closeStmts closes the statements prepared so far
func (q *Queue) closeStmts() {
- for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.ack, q.reject, q.count, q.countCapped, q.values} {
+ for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.fin, q.count, q.countCapped, q.values, q.putJob, q.unmark, q.hasMark, q.isMarked} {
if s != nil {
s.Close()
}
@@ -349,12 +364,12 @@ func (q *Queue) enqueue(item any, priority int, id string) bool {
// claimN moves up to n pending rows to processing in one statement and
// returns them in queue order
-func (q *Queue) claimN(n int) ([]claimed, error) {
+func (q *Queue) claimN(n int) ([]claimed, bool, error) {
q.f.mu.Lock()
- out, err := q.claimRows(n)
+ out, capped, err := q.claimGated(n)
q.f.mu.Unlock()
if err != nil {
- return nil, err
+ return nil, false, err
}
// RETURNING order is unspecified
@@ -364,13 +379,61 @@ func (q *Queue) claimN(n int) ([]claimed, error) {
}
return cmp.Compare(a.id, b.id)
})
- return out, nil
+ return out, capped, nil
+}
+
+// claimGated claims up to n rows; with a live cancel mark on the file it
+// runs in one transaction that drops each marked row it claims (outcome
+// cancelled) and claims again, at most maxDrops drops; capped reports that
+// it stopped there. The caller holds the write mutex.
+func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) {
+ now := time.Now()
+ if !q.anyMark(now) {
+ out, err = q.claimRows(q.claim, n)
+ return out, false, err
+ }
+
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return nil, false, err
+ }
+ defer tx.Rollback()
+
+ claim, isMarked := tx.Stmt(q.claim), tx.Stmt(q.isMarked)
+ out = make([]claimed, 0, n)
+ drops := 0
+ for len(out) < n && drops < maxDrops {
+ // No more than the drops left: a batch never passes the cap
+ rows, err := q.claimRows(claim, min(n-len(out), maxDrops-drops))
+ if err != nil {
+ return nil, false, err
+ }
+ if len(rows) == 0 {
+ break
+ }
+ for _, c := range rows {
+ id := jobIDOf(c.once, c.data)
+ drop, err := marked(isMarked, id, now)
+ if err != nil {
+ return nil, false, err
+ }
+ if !drop {
+ out = append(out, c)
+ continue
+ }
+ if _, err := q.end(tx, c.id, id, contract.JobCancelled, "cancelled", nil, now); err != nil {
+ return nil, false, err
+ }
+ drops++
+ }
+ }
+ return out, len(out) < n && drops >= maxDrops, tx.Commit()
}
-// claimRows runs the claim statement and reads every row it returns; the
+// claimRows runs a claim statement and reads every row it returns; the
// caller holds the write mutex
-func (q *Queue) claimRows(n int) ([]claimed, error) {
- rows, err := q.claim.Query(time.Now().UTC(), n)
+func (q *Queue) claimRows(claim *sql.Stmt, n int) ([]claimed, error) {
+ rows, err := claim.Query(time.Now().UTC(), n)
if err != nil {
return nil, err
}
@@ -380,9 +443,9 @@ func (q *Queue) claimRows(n int) ([]claimed, error) {
for rows.Next() {
var c claimed
if q.priority {
- err = rows.Scan(&c.id, &c.priority, &c.data)
+ err = rows.Scan(&c.id, &c.priority, &c.data, &c.once)
} else {
- err = rows.Scan(&c.id, &c.data)
+ err = rows.Scan(&c.id, &c.data, &c.once)
}
if err != nil {
return nil, err
@@ -403,25 +466,57 @@ func (q *Queue) Dequeue() (any, bool) {
q.f.mu.Lock()
defer q.f.mu.Unlock()
- // Deferred last, so the rows close (and the statement ends) before unlocking
- rows, err := q.pop.Query()
- if err != nil {
- return nil, false
+ now := time.Now()
+ if !q.anyMark(now) {
+ data, _, err := popRow(q.pop)
+ return data, err == nil
}
- defer rows.Close()
-
- var data []byte
- found := false
- for rows.Next() {
- if err := rows.Scan(&data); err != nil {
- return nil, false
+ for {
+ data, capped, err := q.popGated(now)
+ if err != nil || data != nil || !capped {
+ return data, err == nil && data != nil
}
- found = true
}
- if rows.Err() != nil || !found {
- return nil, false
+}
+
+// popGated pops in one transaction, dropping each marked item it pops
+// (outcome cancelled), at most maxDrops; capped reports it stopped there
+// with nothing. The caller holds the write mutex.
+func (q *Queue) popGated(now time.Time) (data []byte, capped bool, err error) {
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return nil, false, err
+ }
+ defer tx.Rollback()
+ pop, isMarked := tx.Stmt(q.pop), tx.Stmt(q.isMarked)
+ for range maxDrops {
+ data, once, err := popRow(pop)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, false, tx.Commit() // keep the drops
+ }
+ if err != nil {
+ return nil, false, err
+ }
+ id := jobIDOf(once, data)
+ drop, err := marked(isMarked, id, now)
+ if err != nil {
+ return nil, false, err
+ }
+ if !drop {
+ return data, false, tx.Commit()
+ }
+ if err := q.record(tx, id, contract.JobCancelled, "cancelled", nil, now); err != nil {
+ return nil, false, err
+ }
}
- return data, true
+ return nil, true, tx.Commit()
+}
+
+// popRow runs a pop statement: the deleted row's data and job_id, or
+// sql.ErrNoRows on an empty queue
+func popRow(pop *sql.Stmt) (data []byte, once sql.NullString, err error) {
+ err = pop.QueryRow().Scan(&data, &once)
+ return data, once, err
}
// DequeueWithAckId moves the next item to processing and returns it with its
@@ -431,11 +526,17 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) {
return nil, false, ""
}
- c, err := q.claimN(1)
- if err != nil || len(c) == 0 {
- return nil, false, ""
+ // A pop stopped at the drop cap has nothing yet: pop again, it can't
+ // report items left
+ for {
+ c, capped, err := q.claimN(1)
+ if err != nil || len(c) == 0 && !capped {
+ return nil, false, ""
+ }
+ if len(c) > 0 {
+ return c[0].data, true, strconv.FormatInt(c[0].id, 10)
+ }
}
- return c[0].data, true, strconv.FormatInt(c[0].id, 10)
}
// DequeueBatchWithAckIds implements varmq.IBatchAcknowledgeable: it moves up
@@ -449,12 +550,17 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini
return 0, 0, nil
}
- c, err := q.claimN(want)
+ c, capped, err := q.claimN(want)
if err != nil {
return 0, 0, err
}
+ // At the drop cap items may wait still: report one, so the caller pops again
+ left := 0
+ if capped {
+ left = 1
+ }
if len(c) == 0 {
- return 0, 0, nil
+ return 0, left, nil
}
for i, row := range c {
items[i], ackIDs[i] = row.data, strconv.FormatInt(row.id, 10)
@@ -465,52 +571,83 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini
if err := q.countCapped.QueryRow().Scan(&remaining); err != nil {
return len(c), 1, nil
}
- return len(c), remaining, nil
+ return len(c), max(remaining, left), nil
}
// Acknowledge marks a processing item as completed, deleting it unless
// WithRemoveOnComplete(false) is set, in the file's next group commit round
// Returns true only if exactly one processing row had this ack ID
-func (q *Queue) Acknowledge(ackID string) bool { return q.finish(ackID, q.ack) }
+func (q *Queue) Acknowledge(ackID string) bool {
+ return q.finish(ackID, contract.JobCompleted, "", nil)
+}
// Reject marks a processing item as failed for good: it is not requeued.
// The row is deleted, or with WithRemoveOnComplete(false) kept with status
-// 'failed'. Same group commit and result as Acknowledge.
-// ponytail: reason is not stored (no column); add one when something reads it.
-func (q *Queue) Reject(ackID string, reason string) bool { return q.finish(ackID, q.reject) }
+// 'failed'. reason is the error of the job's outcome. Same group commit and
+// result as Acknowledge.
+func (q *Queue) Reject(ackID string, reason string) bool {
+ return q.finish(ackID, contract.JobFailed, reason, nil)
+}
-// finish runs stmt (ack or reject) for the row of ackID.
-func (q *Queue) finish(ackID string, stmt *sql.Stmt) bool {
+// finish ends the processing row of ackID with state in the file's next
+// group commit round. A round of one runs in its own transaction: the row
+// and its outcome record commit together.
+func (q *Queue) finish(ackID string, state contract.JobState, reason string, result []byte) bool {
id, ok := parseAckID(ackID)
if !ok {
return false
}
- now := time.Now().UTC()
- return q.f.b.do(func(tx *sql.Tx) (bool, error) {
- var (
- result sql.Result
- err error
- )
- s := stmt
- if tx != nil {
- s = tx.Stmt(s)
- }
- if q.removeOnComplete {
- result, err = s.Exec(id)
- } else {
- result, err = s.Exec(now, id)
+ now := time.Now()
+ return q.f.b.do(func(tx *sql.Tx) (ok bool, err error) {
+ if tx == nil {
+ if tx, err = q.f.db.Begin(); err != nil {
+ return false, err
+ }
+ defer func() {
+ if err == nil {
+ err = tx.Commit()
+ } else {
+ tx.Rollback()
+ }
+ ok = ok && err == nil
+ }()
}
- if err != nil {
- return false, err
- }
-
// No row is an outcome, not an error: it must not fail the round
- n, err := result.RowsAffected()
- return err == nil && n == 1, err
+ return q.end(tx, id, "", state, reason, result, now)
})
}
+// end finishes processing row id with state, in tx: the row is deleted or
+// kept with state as status, and a job with a Job ID gets its outcome
+// record. jobID is the row's, when the caller knows it. Reports whether
+// the row was processing.
+func (q *Queue) end(tx *sql.Tx, id int64, jobID string, state contract.JobState, reason string, result []byte, now time.Time) (bool, error) {
+ var row *sql.Row
+ if q.removeOnComplete {
+ row = tx.Stmt(q.fin).QueryRow(id)
+ } else {
+ row = tx.Stmt(q.fin).QueryRow(string(state), now.UTC(), id)
+ }
+ var (
+ once sql.NullString
+ data []byte
+ )
+ switch err := row.Scan(&once, &data); {
+ case errors.Is(err, sql.ErrNoRows):
+ return false, nil
+ case err != nil:
+ return false, err
+ }
+ if jobID == "" {
+ jobID = jobIDOf(once, data)
+ }
+ if jobID == "" {
+ return true, nil
+ }
+ return true, q.record(tx, jobID, state, reason, result, now)
+}
+
// Len returns the number of pending items in the queue
func (q *Queue) Len() int {
var count int
@@ -543,17 +680,30 @@ func (q *Queue) Values() []any {
return items
}
-// Purge removes all items from the queue
+// Purge removes all items from the queue, with its outcome records and
+// cancel marks (spec 6.6, 6.14)
func (q *Queue) Purge() {
q.f.mu.Lock()
defer q.f.mu.Unlock()
- q.f.db.Exec(fmt.Sprintf("DELETE FROM %s", q.table))
+ tx, err := q.f.db.Begin()
+ if err != nil {
+ return
+ }
+ defer tx.Rollback()
+ for _, t := range []string{q.table, q.jobs, q.cancelled} {
+ if _, err := tx.Exec("DELETE FROM " + t); err != nil {
+ return
+ }
+ }
+ tx.Commit()
}
-// Close closes the queue and its database connection
+// Close closes the queue and drops its outcome watches and OnCancel hooks
+// without calling them. The database file stays open for the Queues.
func (q *Queue) Close() error {
q.closed.Store(true)
+ q.w.close()
return nil
}
diff --git a/queue_test.go b/queue_test.go
index bf66ab0..6f55614 100644
--- a/queue_test.go
+++ b/queue_test.go
@@ -1,14 +1,20 @@
package sqliteq
import (
+ "context"
"database/sql"
+ "errors"
"fmt"
"path/filepath"
+ "slices"
"strconv"
"sync"
"sync/atomic"
"testing"
"time"
+
+ "github.com/goptics/varmq"
+ "github.com/goptics/varmq/contract"
)
func TestSQLiteQueue(t *testing.T) {
@@ -844,6 +850,9 @@ func TestWritesTakeFileMutex(t *testing.T) {
{"DequeueWithAckId", func() { q.DequeueWithAckId() }},
{"DequeueBatchWithAckIds", func() { q.DequeueBatchWithAckIds(make([]any, 2), make([]string, 2)) }},
{"Acknowledge", func() { q.Acknowledge(ack1) }},
+ {"CancelJob", func() { q.CancelJob("x") }},
+ {"gated DequeueWithAckId", func() { q.DequeueWithAckId() }},
+ {"gated Dequeue", func() { q.Dequeue() }},
{"requeueNoAck", func() { q.requeueNoAck() }},
{"Purge", func() { q.Purge() }},
}
@@ -894,3 +903,777 @@ func TestWritesTakeFileMutex(t *testing.T) {
})
}
}
+
+// otherProcess opens q's table again on a pool of its own, as a second
+// process on the same database file would.
+func otherProcess(t *testing.T, q *Queue) *Queue {
+ t.Helper()
+ db, err := sql.Open(driverName(q.f.pages), dsn(q.f.key))
+ if err != nil {
+ t.Fatal(err)
+ }
+ f := &file{key: q.f.key, db: db, pages: q.f.pages, refs: 1, handles: map[string]*handle{}, recovered: map[string]bool{}}
+ f.b = &batcher{commit: f.commitRound}
+ q2, err := newQueue(f, q.name, q.priority, WithRemoveOnComplete(q.removeOnComplete))
+ if err != nil {
+ db.Close()
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ q2.Close()
+ db.Close()
+ })
+ return q2
+}
+
+// job is a varmq-style envelope with Job ID id.
+func job(id string) []byte { return []byte(`{"id":"` + id + `","data":1}`) }
+
+// watch starts a watch of id on q; the channel gets its one outcome.
+func watch(q *Queue, id string) <-chan contract.JobOutcome {
+ ch := make(chan contract.JobOutcome, 2)
+ q.WatchOutcome(id, func(o contract.JobOutcome) { ch <- o })
+ return ch
+}
+
+func waitOutcome(t *testing.T, ch <-chan contract.JobOutcome) contract.JobOutcome {
+ t.Helper()
+ select {
+ case o := <-ch:
+ return o
+ case <-time.After(5 * time.Second):
+ t.Fatal("no outcome")
+ return contract.JobOutcome{}
+ }
+}
+
+func noOutcome(t *testing.T, ch <-chan contract.JobOutcome) {
+ t.Helper()
+ select {
+ case o := <-ch:
+ t.Fatalf("unexpected outcome %+v", o)
+ case <-time.After(4 * pollEvery):
+ }
+}
+
+// pop hands out the next item and its ack id, failing on none.
+func pop(t *testing.T, q *Queue) (string, string) {
+ t.Helper()
+ v, ok, ack := q.DequeueWithAckId()
+ if !ok {
+ t.Fatal("DequeueWithAckId found nothing")
+ }
+ return string(v.([]byte)), ack
+}
+
+// A watch reports each way a job ends: Acknowledge, Reject with its reason,
+// AcknowledgeResult with its JSON; a job without a Job ID writes nothing.
+func TestWatchOutcome(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+
+ cases := []struct {
+ id string
+ finish func(ack string) bool
+ want contract.JobOutcome
+ }{
+ {"ack", q.Acknowledge, contract.JobOutcome{ID: "ack", State: contract.JobCompleted}},
+ {"rej", func(a string) bool { return q.Reject(a, "boom") }, contract.JobOutcome{ID: "rej", State: contract.JobFailed, Error: "boom"}},
+ {"res", func(a string) bool { return q.AcknowledgeResult(a, []byte(`{"n":42}`)) },
+ contract.JobOutcome{ID: "res", State: contract.JobCompleted, Result: []byte(`{"n":42}`)}},
+ }
+ for _, c := range cases {
+ ch := watch(q, c.id)
+ q.Enqueue(job(c.id))
+ _, ack := pop(t, q)
+ noOutcome(t, ch)
+ if !c.finish(ack) {
+ t.Fatalf("%s: finish failed", c.id)
+ }
+ got := waitOutcome(t, ch)
+ if got.ID != c.want.ID || got.State != c.want.State || got.Error != c.want.Error || string(got.Result) != string(c.want.Result) {
+ t.Errorf("%s: got %+v, want %+v", c.id, got, c.want)
+ }
+ noOutcome(t, ch) // once
+ }
+
+ q.Enqueue([]byte("plain"))
+ _, ack := pop(t, q)
+ q.Acknowledge(ack)
+ if n := countJobs(t, q); n != 3 {
+ t.Errorf("Expected 3 outcome records, got %d", n)
+ }
+}
+
+func countJobs(t *testing.T, q *Queue) int {
+ t.Helper()
+ var n int
+ if err := q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.jobs).Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ return n
+}
+
+// AcknowledgeResult with invalid JSON fails and changes nothing.
+func TestAcknowledgeResultInvalid(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("a"))
+ _, ack := pop(t, q)
+ if q.AcknowledgeResult(ack, []byte(`{nope`)) {
+ t.Fatal("Expected invalid JSON to fail")
+ }
+ if n := countRows(t, q.f.db, "status = 'processing'"); n != 1 {
+ t.Errorf("Expected the row still processing, got %d", n)
+ }
+ if !q.Acknowledge(ack) {
+ t.Error("Expected Acknowledge to finish it")
+ }
+}
+
+// A job that ended before the watch is reported from its record; cancel
+// stops a watch; a closed queue watches nothing.
+func TestWatchEndedBeforeAndCancel(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("early"))
+ _, ack := pop(t, q)
+ q.Reject(ack, "late watcher")
+ if got := waitOutcome(t, watch(q, "early")); got.State != contract.JobFailed || got.Error != "late watcher" {
+ t.Errorf("got %+v", got)
+ }
+
+ ch := make(chan contract.JobOutcome, 1)
+ stop := q.WatchOutcome("never", func(o contract.JobOutcome) { ch <- o })
+ stop()
+ stop()
+ q.Enqueue(job("never"))
+ _, ack = pop(t, q)
+ q.Acknowledge(ack)
+ noOutcome(t, ch)
+
+ q.Close()
+ q.WatchOutcome("early", func(o contract.JobOutcome) { ch <- o })
+ noOutcome(t, ch)
+ if err := q.CancelJob("x"); !errors.Is(err, errClosed) {
+ t.Errorf("CancelJob on a closed queue: got %v, want errClosed", err)
+ }
+}
+
+// Two queue objects on one file, as two processes: each learns of jobs the
+// other finishes, and a cancel reaches the other's pops and OnCancel.
+func TestOutcomeAcrossProcesses(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q2 := otherProcess(t, q)
+
+ ch := watch(q, "x")
+ q.Enqueue(job("x"))
+ _, ack := pop(t, q2)
+ q2.Acknowledge(ack)
+ if got := waitOutcome(t, ch); got.ID != "x" || got.State != contract.JobCompleted {
+ t.Errorf("got %+v", got)
+ }
+
+ ids := make(chan string, 4)
+ defer q2.OnCancel(func(id string) { ids <- id })()
+ q2.Enqueue(job("run"))
+ _, ack = pop(t, q2)
+ ch = watch(q2, "run")
+ if err := q.CancelJob("run"); err != nil {
+ t.Fatal(err)
+ }
+ select {
+ case id := <-ids:
+ if id != "run" {
+ t.Errorf("OnCancel got %q", id)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("OnCancel did not fire")
+ }
+ if !q2.AcknowledgeCancelled(ack, "stopped") {
+ t.Fatal("AcknowledgeCancelled failed")
+ }
+ if got := waitOutcome(t, ch); got.State != contract.JobCancelled || got.Error != "stopped" {
+ t.Errorf("got %+v", got)
+ }
+ var marks int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks)
+ if marks != 0 {
+ t.Errorf("Expected the finish to clear the mark, %d left", marks)
+ }
+ if err := q.CancelJob("run"); !errors.Is(err, contract.ErrJobEnded) {
+ t.Errorf("CancelJob of an ended job: got %v, want ErrJobEnded", err)
+ }
+}
+
+// A marked waiting job is never handed out, by any pop: it is dropped with
+// outcome cancelled and the pop goes on with the next item.
+func TestCancelWaiting(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(false))
+ defer qs.Close()
+
+ for _, id := range []string{"a", "b", "c", "d", "e"} {
+ q.Enqueue(job(id))
+ }
+ q.EnqueueOnce([]byte("raw"), "f") // a Job ID from EnqueueOnce, not JSON
+ q.Enqueue([]byte("g"))
+ chA, chF := watch(q, "a"), watch(q, "f")
+ for _, id := range []string{"a", "c", "d", "f"} {
+ if err := q.CancelJob(id); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if v, _ := pop(t, q); v != string(job("b")) {
+ t.Errorf("DequeueWithAckId: got %s, want b", v)
+ }
+ items, acks := make([]any, 2), make([]string, 2)
+ n, _, err := q.DequeueBatchWithAckIds(items, acks)
+ if err != nil || n != 2 || string(items[0].([]byte)) != string(job("e")) || string(items[1].([]byte)) != "g" {
+ t.Errorf("DequeueBatchWithAckIds: got %d %v %v", n, err, items[:n])
+ }
+ if got := waitOutcome(t, chA); got.State != contract.JobCancelled || got.Error != "cancelled" {
+ t.Errorf("got %+v", got)
+ }
+ if got := waitOutcome(t, chF); got.State != contract.JobCancelled {
+ t.Errorf("got %+v", got)
+ }
+ if n := countRows(t, q.f.db, "status = 'cancelled'"); n != 4 {
+ t.Errorf("Expected 4 kept cancelled rows, got %d", n)
+ }
+ if _, ok, _ := q.DequeueWithAckId(); ok {
+ t.Error("Expected nothing left")
+ }
+
+ // Dequeue without ack drops too
+ q.Enqueue(job("h"))
+ q.Enqueue(job("i"))
+ q.CancelJob("h")
+ if v, ok := q.Dequeue(); !ok || string(v.([]byte)) != string(job("i")) {
+ t.Errorf("Dequeue: got %v %v, want i", v, ok)
+ }
+ if got := waitOutcome(t, watch(q, "h")); got.State != contract.JobCancelled {
+ t.Errorf("got %+v", got)
+ }
+}
+
+// EnqueueOnce that stores an id drops its old outcome and its cancel mark
+// (spec 6.13, 6.14 Re-add); a dedupe hit changes neither.
+func TestReaddClearsOutcomeAndMark(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+
+ q.EnqueueOnce(job("r"), "r")
+ _, ack := pop(t, q)
+ q.Acknowledge(ack)
+ if countJobs(t, q) != 1 {
+ t.Fatal("Expected a record")
+ }
+ q.EnqueueOnce(job("r"), "r")
+ if countJobs(t, q) != 0 {
+ t.Error("Expected the re-add to drop the old record")
+ }
+ ch := watch(q, "r")
+ noOutcome(t, ch) // the earlier run is not reported
+
+ if err := q.CancelJob("r"); err != nil {
+ t.Fatal(err)
+ }
+ q.EnqueueOnce(job("r"), "r") // dedupe hit: the row is still there
+ if _, ok, _ := q.DequeueWithAckId(); ok {
+ t.Fatal("Expected the marked job dropped")
+ }
+ if got := waitOutcome(t, ch); got.State != contract.JobCancelled {
+ t.Errorf("got %+v", got)
+ }
+
+ q.CancelJob("s") // unknown id: marked
+ q.EnqueueOnce(job("s"), "s")
+ if v, _ := pop(t, q); v != string(job("s")) {
+ t.Errorf("Expected the re-added job handed out, got %s", v)
+ }
+}
+
+// Purge drops outcome records and cancel marks with the items.
+func TestPurgeClearsOutcomesAndMarks(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("done"))
+ _, ack := pop(t, q)
+ q.Acknowledge(ack)
+ q.CancelJob("later")
+ q.Purge()
+ var marks int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks)
+ if countJobs(t, q) != 0 || marks != 0 {
+ t.Errorf("Expected Purge to clear records and marks, got %d and %d", countJobs(t, q), marks)
+ }
+ q.Enqueue(job("later"))
+ if v, _ := pop(t, q); v != string(job("later")) {
+ t.Errorf("Expected the purged mark gone, got %s", v)
+ }
+}
+
+// OnCancel reports marks made after it, not before; stop ends it; an
+// expired mark neither reaches hooks nor drops items.
+func TestOnCancel(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.CancelJob("old")
+ ids := make(chan string, 4)
+ stop := q.OnCancel(func(id string) { ids <- id })
+ q.CancelJob("new")
+ select {
+ case id := <-ids:
+ if id != "new" {
+ t.Errorf("got %q, want new", id)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("OnCancel did not fire")
+ }
+ stop()
+ q.CancelJob("after")
+ select {
+ case id := <-ids:
+ t.Errorf("got %q after stop", id)
+ case <-time.After(4 * pollEvery):
+ }
+
+ q.f.db.Exec("UPDATE "+q.cancelled+" SET expires_at = ? WHERE job_id = 'old'", time.Now().Add(-time.Second).UnixMilli())
+ q.Enqueue(job("old"))
+ if v, _ := pop(t, q); v != string(job("old")) {
+ t.Errorf("Expected an expired mark to drop nothing, got %s", v)
+ }
+ if err := q.CancelJob(""); err == nil {
+ t.Error("Expected an empty id to fail")
+ }
+}
+
+// Retention trims each state group by age and count; the count limit
+// spares records younger than trimFloor; a queue object's first record
+// trims.
+func TestOutcomeRetention(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ now := time.Now()
+ insert := func(tx *sql.Tx, prefix string, n int, at time.Time) {
+ for i := range n {
+ if _, err := tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES (?, 'completed', ?)", prefix+strconv.Itoa(i), at.UnixMilli()+int64(i)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+
+ tx, _ := q.f.db.Begin()
+ insert(tx, "fresh", keepCompleted+5, now)
+ if err := q.trim(tx, now); err != nil {
+ t.Fatal(err)
+ }
+ tx.Commit()
+ if n := countJobs(t, q); n != keepCompleted+5 {
+ t.Fatalf("Expected records younger than trimFloor kept, got %d", n)
+ }
+ q.f.db.Exec("DELETE FROM " + q.jobs)
+
+ tx, _ = q.f.db.Begin()
+ insert(tx, "c", keepCompleted+5, now.Add(-2*trimFloor))
+ tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", now.Add(-keepFailedAge).UnixMilli())
+ tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'cancelled', ?)", now.UnixMilli())
+ if err := q.trim(tx, now); err != nil {
+ t.Fatal(err)
+ }
+ tx.Commit()
+ if n := countJobs(t, q); n != keepCompleted+1 {
+ t.Errorf("Expected %d records, got %d", keepCompleted+1, n)
+ }
+ var oldest string
+ q.f.db.QueryRow("SELECT job_id FROM " + q.jobs + " WHERE state = 'completed' ORDER BY finished_at LIMIT 1").Scan(&oldest)
+ if oldest != "c5" {
+ t.Errorf("Expected the 5 oldest completed trimmed, oldest left %s", oldest)
+ }
+
+ q2 := otherProcess(t, q) // records counter at 0
+ q2.f.db.Exec("UPDATE "+q.jobs+" SET finished_at = ? WHERE job_id = 'c5'", now.Add(-keepCompletedAge).UnixMilli())
+ q2.Enqueue(job("one"))
+ _, ack := pop(t, q2)
+ q2.Acknowledge(ack)
+ var n int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.jobs + " WHERE job_id = 'c5'").Scan(&n)
+ if n != 0 {
+ t.Error("Expected a queue object's first record to trim")
+ }
+}
+
+// varmq end to end: a result job added on one queue object runs on a
+// worker bound to another (another process), and its handle gets the
+// result; Cancel stops a running job and drops a waiting one.
+func TestVarmqRemoteResultAndCancel(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q2 := otherProcess(t, q)
+
+ started := make(chan struct{})
+ ran := make(chan int, 8)
+ w := varmq.NewResultWorker(func(j varmq.Job[int]) (int, error) {
+ ran <- j.Data()
+ if j.Data() < 0 {
+ close(started)
+ <-j.Context().Done()
+ return 0, j.Context().Err()
+ }
+ return j.Data() * 2, nil
+ })
+ defer w.Stop()
+
+ p := varmq.NewResultQueue[int, int](q)
+ waiting, ok := p.Add(7) // no worker yet
+ if !ok {
+ t.Fatal("Add failed")
+ }
+ if err := waiting.Cancel(); err != nil {
+ t.Fatal(err)
+ }
+ w.WithQueue(q2)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ j, _ := p.Add(21)
+ if err := j.WaitContext(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if r, err := j.Result(); err != nil || r != 42 {
+ t.Errorf("Result: got %d %v, want 42", r, err)
+ }
+ if err := waiting.WaitContext(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := waiting.Result(); !errors.Is(err, varmq.ErrJobCancelled) {
+ t.Errorf("waiting job: got %v, want ErrJobCancelled", err)
+ }
+
+ running, _ := p.Add(-1)
+ select {
+ case <-started:
+ case <-ctx.Done():
+ t.Fatal("job did not start")
+ }
+ if err := running.Cancel(); err != nil {
+ t.Fatal(err)
+ }
+ if err := running.WaitContext(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := running.Result(); !errors.Is(err, varmq.ErrJobCancelled) {
+ t.Errorf("running job: got %v, want ErrJobCancelled", err)
+ }
+ close(ran)
+ for v := range ran {
+ if v == 7 {
+ t.Error("The cancelled waiting job ran")
+ }
+ }
+}
+
+// A pop drops at most maxDrops marked items; at the cap it reports items
+// left, and the next pop goes on.
+func TestCancelDropCap(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ for i := range maxDrops {
+ id := "m" + strconv.Itoa(i)
+ q.Enqueue(job(id))
+ q.CancelJob(id)
+ }
+ q.Enqueue([]byte("plain"))
+
+ items, acks := make([]any, 1), make([]string, 1)
+ if n, left, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != 0 || left != 1 {
+ t.Fatalf("at the cap: got n=%d left=%d err=%v, want 0, 1", n, left, err)
+ }
+ if v, _ := pop(t, q); v != "plain" {
+ t.Errorf("got %s, want plain", v)
+ }
+ if n := countJobs(t, q); n != maxDrops {
+ t.Errorf("Expected %d cancelled records, got %d", maxDrops, n)
+ }
+
+ // Single-item pops can't report items left: past the cap they pop again
+ for _, p := range []struct {
+ name string
+ pop func() (any, bool)
+ }{
+ {"DequeueWithAckId", func() (any, bool) { v, ok, _ := q.DequeueWithAckId(); return v, ok }},
+ {"Dequeue", q.Dequeue},
+ } {
+ for i := range maxDrops + 1 {
+ id := p.name + strconv.Itoa(i)
+ q.Enqueue(job(id))
+ q.CancelJob(id)
+ }
+ q.Enqueue([]byte("plain"))
+ if v, ok := p.pop(); !ok || string(v.([]byte)) != "plain" {
+ t.Errorf("%s past the cap: got %v %v, want plain", p.name, v, ok)
+ }
+ }
+}
+
+// sweep trims records past their age and drops expired marks.
+func TestSweep(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ old := time.Now().Add(-keepFailedAge).UnixMilli()
+ q.f.db.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", old)
+ q.f.db.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'failed', ?)", time.Now().UnixMilli())
+ q.f.db.Exec("INSERT INTO "+q.cancelled+" (job_id, expires_at) VALUES ('gone', ?)", old)
+ q.CancelJob("live")
+ q.sweep()
+ var marks int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks)
+ if countJobs(t, q) != 1 || marks != 1 {
+ t.Errorf("Expected 1 record and 1 mark left, got %d and %d", countJobs(t, q), marks)
+ }
+}
+
+// Queues.Close stops polls and drops live watches and hooks without
+// calling them.
+func TestCloseDropsWatchesAndHooks(t *testing.T) {
+ q, qs := openQueue(t)
+ q2 := otherProcess(t, q)
+ ch := watch(q, "w")
+ ids := make(chan string, 1)
+ q.OnCancel(func(id string) { ids <- id })
+ if err := qs.Close(); err != nil {
+ t.Fatal(err)
+ }
+ select {
+ case <-q.w.stop:
+ default:
+ t.Fatal("Expected Close to stop the poll")
+ }
+ q2.Enqueue(job("w"))
+ _, ack := pop(t, q2)
+ q2.Acknowledge(ack)
+ q2.CancelJob("c")
+ noOutcome(t, ch)
+ select {
+ case id := <-ids:
+ t.Errorf("hook called after Close with %q", id)
+ default:
+ }
+ if stop := q.OnCancel(func(string) { t.Error("hook on a closed queue") }); stop == nil {
+ t.Error("Expected a stop func")
+ }
+}
+
+// A hook added while a poll reads gets the marks made after its call; a
+// stopped hook gets none; a mark whose job finished (stale) reaches nobody.
+func TestOnCancelHooksDuringPoll(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ a, b := make(chan string, 4), make(chan string, 4)
+ stopA := q.OnCancel(func(id string) { a <- id })
+ q.w.mu.Lock()
+ seq := q.w.lastSeq // a poll's snapshot, taken before B
+ q.w.mu.Unlock()
+ q.CancelJob("before-b")
+ stopB := q.OnCancel(func(id string) { b <- id })
+ q.CancelJob("x")
+ q.pollCancels(seq)
+
+ // The queue's own poll may report a mark too: compare distinct ids
+ got := func(ch chan string) []string {
+ var ids []string
+ for {
+ select {
+ case id := <-ch:
+ ids = append(ids, id)
+ case <-time.After(4 * pollEvery):
+ slices.Sort(ids)
+ return slices.Compact(ids)
+ }
+ }
+ }
+ if ids := got(a); !slices.Equal(ids, []string{"before-b", "x"}) {
+ t.Errorf("hook A: got %v", ids)
+ }
+ if ids := got(b); !slices.Equal(ids, []string{"x"}) {
+ t.Errorf("hook B: got %v, want only the mark after its call", ids)
+ }
+
+ var xSeq int64
+ q.f.db.QueryRow("SELECT seq FROM " + q.cancelled + " WHERE job_id = 'x'").Scan(&xSeq)
+ stopA()
+ q.Enqueue(job("y"))
+ _, ack := pop(t, q)
+ q.CancelJob("y")
+ q.Acknowledge(ack) // the finish drops the mark: stale
+ q.CancelJob("z")
+ q.pollCancels(xSeq)
+ if ids := got(a); len(ids) != 0 {
+ t.Errorf("stopped hook got %v", ids)
+ }
+ if ids := got(b); !slices.Equal(ids, []string{"z"}) {
+ t.Errorf("hook B: got %v, want z only (y is stale)", ids)
+ }
+ stopB()
+}
+
+// A repeat CancelJob gets a new seq and expiry, so hooks hear it again.
+func TestCancelRepeatRenotifies(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ ids := make(chan string, 4)
+ defer q.OnCancel(func(id string) { ids <- id })()
+ read := func() (seq, exp int64) {
+ q.f.db.QueryRow("SELECT seq, expires_at FROM "+q.cancelled+" WHERE job_id = 'x'").Scan(&seq, &exp)
+ return
+ }
+ for i := range 2 {
+ q.CancelJob("x")
+ select {
+ case <-ids:
+ case <-time.After(5 * time.Second):
+ t.Fatalf("cancel %d not reported", i+1)
+ }
+ if i == 0 {
+ s1, e1 := read()
+ time.Sleep(2 * time.Millisecond)
+ defer func() {
+ if s2, e2 := read(); s2 <= s1 || e2 <= e1 {
+ t.Errorf("Expected a new seq and expiry: %d,%d -> %d,%d", s1, e1, s2, e2)
+ }
+ }()
+ }
+ }
+}
+
+// A gated Dequeue that drops the last item returns empty and keeps the
+// drop; the mark is gone after it.
+func TestDequeueGatedEmpty(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.Enqueue(job("x"))
+ q.CancelJob("x")
+ ch := watch(q, "x")
+ if v, ok := q.Dequeue(); ok {
+ t.Fatalf("got %v", v)
+ }
+ if got := waitOutcome(t, ch); got.State != contract.JobCancelled {
+ t.Errorf("got %+v", got)
+ }
+ var marks int
+ q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks)
+ if q.Len() != 0 || marks != 0 {
+ t.Errorf("Expected no item and no mark, got %d and %d", q.Len(), marks)
+ }
+}
+
+// With WithRemoveOnComplete(false) each end keeps its row with its status,
+// and the record keeps the result.
+func TestKeptRowEnds(t *testing.T) {
+ q, qs := openQueue(t, WithRemoveOnComplete(false))
+ defer qs.Close()
+ q.Enqueue(job("r"))
+ q.Enqueue(job("c"))
+ _, ackR := pop(t, q)
+ _, ackC := pop(t, q)
+ if !q.AcknowledgeResult(ackR, []byte(`"ok"`)) || !q.AcknowledgeCancelled(ackC, "stop") {
+ t.Fatal("finish failed")
+ }
+ if countRows(t, q.f.db, "status = 'completed'") != 1 || countRows(t, q.f.db, "status = 'cancelled'") != 1 {
+ t.Error("Expected one completed and one cancelled row")
+ }
+ if got := waitOutcome(t, watch(q, "r")); string(got.Result) != `"ok"` {
+ t.Errorf("got %+v", got)
+ }
+ if got := waitOutcome(t, watch(q, "c")); got.State != contract.JobCancelled || got.Error != "stop" {
+ t.Errorf("got %+v", got)
+ }
+}
+
+// A read reaches only watches started before it: an earlier run's record,
+// read before a re-add, never ends a watch made after.
+func TestDeliverSkipsNewerWatches(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ ch := watch(q, "x")
+ q.w.mu.Lock()
+ gen := q.w.gen
+ q.w.mu.Unlock()
+ q.deliver(contract.JobOutcome{ID: "x", State: contract.JobCompleted}, gen-1)
+ noOutcome(t, ch)
+ q.deliver(contract.JobOutcome{ID: "x", State: contract.JobCompleted}, gen)
+ waitOutcome(t, ch)
+}
+
+// Every watch fires when far more jobs end between two polls than
+// retention keeps, and more ids are watched than one read used to take.
+func TestWatchBurst(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q2 := otherProcess(t, q)
+ const n = 3 * keepCompleted
+ var fired atomic.Int64
+ done := make(chan struct{})
+ for i := range n {
+ id := "b" + strconv.Itoa(i)
+ q.WatchOutcome(id, func(contract.JobOutcome) {
+ if fired.Add(1) == n {
+ close(done)
+ }
+ })
+ q.Enqueue(job(id))
+ }
+ items, acks := make([]any, 500), make([]string, 500)
+ for {
+ k, _, err := q2.DequeueBatchWithAckIds(items, acks)
+ if err != nil || k == 0 {
+ break
+ }
+ for _, a := range acks[:k] {
+ q2.Acknowledge(a)
+ }
+ }
+ select {
+ case <-done:
+ case <-time.After(20 * time.Second):
+ t.Fatalf("%d of %d watches fired", fired.Load(), n)
+ }
+}
+
+// Queue names may not hold __sqliteq_, which names a queue's own tables.
+func TestReservedQueueName(t *testing.T) {
+ qs := mustNew(t, filepath.Join(t.TempDir(), "one.db"), WithSingleFile())
+ defer qs.Close()
+ if _, err := qs.NewQueue("orders__sqliteq_jobs"); err == nil {
+ t.Error("Expected a reserved name to fail")
+ }
+}
+
+// An item whose envelope does not decode to a string id is a plain job:
+// with a live mark (gated pops) it is handed out, finishes as usual, and
+// writes no record.
+func TestUndecodableEnvelopeIsPlainJob(t *testing.T) {
+ q, qs := openQueue(t)
+ defer qs.Close()
+ q.CancelJob("other") // keeps the pops gated
+ bad := []string{`{bad`, `{"id":5}`, `{"id":null}`, `{"id":""}`, `[1]`, "plain", `{"id":"x"`}
+ for _, b := range bad {
+ q.Enqueue([]byte(b))
+ }
+ for i, b := range bad {
+ v, ack := pop(t, q)
+ if v != b {
+ t.Fatalf("pop %d: got %q, want %q", i, v, b)
+ }
+ if i%2 == 0 {
+ q.Acknowledge(ack)
+ } else {
+ q.Reject(ack, "r")
+ }
+ }
+ if n := countJobs(t, q); n != 0 {
+ t.Errorf("Expected no records, got %d", n)
+ }
+}
diff --git a/registry.go b/registry.go
index 24aeb0e..5db083a 100644
--- a/registry.go
+++ b/registry.go
@@ -62,8 +62,11 @@ func driverName(pages int) string {
// dsn returns the connection string for a database file: WAL, relaxed
// fsync (safe under WAL) and a busy timeout so writers wait for each other.
+// Transactions begin IMMEDIATE: one that reads before it writes (CancelJob,
+// a gated pop) would otherwise fail with SQLITE_BUSY, not wait, when another
+// process wrote in between.
func dsn(path string) string {
- return path + "?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000"
+ return path + "?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000&_txlock=immediate"
}
// registry holds every open file in the process, keyed by resolved path.
@@ -121,6 +124,10 @@ func (f *file) release() error {
if registry.files[f.key] == f {
delete(registry.files, f.key)
}
+ // Stop the queues' polls before the pool closes under them
+ for _, h := range f.handles {
+ h.queue().Close()
+ }
registry.mu.Unlock()
// Close waits for running queries, so it runs outside the lock
@@ -143,6 +150,10 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error)
}
key := foldName(name)
+ // A queue's indexes, sidecar tables and trigger are named __sqliteq_*
+ if strings.Contains(key, "__sqliteq_") {
+ return nil, fmt.Errorf("sqliteq: queue name %q must not contain __sqliteq_", name)
+ }
if h, ok := f.handles[key]; ok {
if (h.pq != nil) != priority {
return nil, fmt.Errorf("sqliteq: %q is already open as a %s", name, h.kind())