diff --git a/.github/workflows/sqliteq.yml b/.github/workflows/sqliteq.yml index b658b4d..bf9d168 100644 --- a/.github/workflows/sqliteq.yml +++ b/.github/workflows/sqliteq.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: "1.24.1" + go-version: "1.27.0" cache: true - name: Install dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b5eabe..8961d18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is not stored. +- `Reject(ackID, reason)` on `Queue` and `PriorityQueue`, required by varmq's `IAcknowledgeable` since varmq#173: finishes an item that failed for good, never requeued. Deleted like an ack, or with `WithRemoveOnComplete(false)` kept with `status` `failed`. Same group commit as `Acknowledge`. The reason is the job's outcome error. +- Job outcome (varmq spec 6.13): `WatchOutcome` (`contract.OutcomeWatcher`) and `AcknowledgeResult` (`contract.ResultAcknowledgeable`) on both queues. A job with a Job ID (`EnqueueOnce` id, else the item's JSON `id`) gets a record in `__sqliteq_jobs` when it ends, same transaction as the finish. Watches poll it by seq every 50 ms (new records only), from any queue object on the file, any process. Retention: completed 1000 / 24 h, failed and cancelled 10000 / 7 d; the count limit spares records under 1 min old. A polling queue object also sweeps retention and expired marks once a minute. +- Job cancel (varmq spec 6.14): `CancelJob`, `OnCancel`, `AcknowledgeCancelled` (`contract.JobCanceller`, `CancelNotifier`, `CancelAcknowledgeable`). Marks in `__sqliteq_cancelled`, 24 h each. Every pop drops a marked waiting job (outcome `cancelled`, at most 1000 per transaction; a single-item pop goes on past that, a batch pop reports an item left); `OnCancel` polls new marks. `ErrJobEnded` when the record shows it ended. +- `EnqueueOnce` that stores an id drops its old record and mark (trigger). `Purge` clears records and marks too. `Queue.Close` drops watches and hooks. + +### Breaking + +- Requires Go 1.27 (was 1.24), for the varmq dependency. +- A queue name must not contain `__sqliteq_`, in both layouts: a queue's indexes, sidecar tables and trigger use it. + +### Changed + +- Transactions begin `IMMEDIATE`. A finish (`Acknowledge`, `Reject`) always runs in a transaction; a pop runs in one while any cancel mark is live. ## [0.3.0] - 2026-10-23 diff --git a/README.md b/README.md index d911546..6058b8b 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Go Reference CI Codecov - Go Version + Go Version License

@@ -49,7 +49,7 @@ go get github.com/goptics/sqliteq ``` -Requires Go 1.24+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler. +Requires Go 1.27+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler. ## Quick Start diff --git a/cancel.go b/cancel.go new file mode 100644 index 0000000..6b4445e --- /dev/null +++ b/cancel.go @@ -0,0 +1,192 @@ +package sqliteq + +import ( + "database/sql" + "errors" + "fmt" + "time" + + "github.com/goptics/varmq/contract" +) + +// Job cancel (varmq spec/redis-storage.md 1.7.0, 6.14), on SQLite. +// CancelJob writes a mark to __sqliteq_cancelled (expires_at: ms). +// Every pop drops a marked waiting job with outcome cancelled; OnCancel +// learns of marks by polling seq; AcknowledgeCancelled finishes a running +// job that stopped for it. Any finish, and an EnqueueOnce that stores the +// id, clears the mark. sqliteq has no workflow store, so no item is exempt. + +var ( + _ contract.JobCanceller = (*Queue)(nil) + _ contract.JobCanceller = (*PriorityQueue)(nil) + _ contract.CancelNotifier = (*Queue)(nil) + _ contract.CancelNotifier = (*PriorityQueue)(nil) + _ contract.CancelAcknowledgeable = (*Queue)(nil) + _ contract.CancelAcknowledgeable = (*PriorityQueue)(nil) +) + +const ( + // cancelTTL is how long one mark lasts. + cancelTTL = 24 * time.Hour + // maxDrops caps the marked items one pop drops; at the cap it returns + // what it has and the next pop goes on. + maxDrops = 1000 +) + +var ( + errEmptyJobID = errors.New("sqliteq: empty job id") + errClosed = errors.New("sqliteq: queue is closed") +) + +// CancelJob implements contract.JobCanceller: it marks job id cancelled for +// cancelTTL. A waiting job is dropped by the pop that would hand it out +// (outcome cancelled); OnCancel callbacks of every queue object on the file +// learn of a running one, which ends only if its worker stops it. +// contract.ErrJobEnded when its outcome record shows it ended. An unknown +// id is marked too: a later Enqueue of it is dropped, unless EnqueueOnce +// stores it (that clears the mark). +func (q *Queue) CancelJob(id string) error { + if id == "" { + return errEmptyJobID + } + if q.closed.Load() { + return errClosed + } + + now := time.Now().UnixMilli() + q.f.mu.Lock() + defer q.f.mu.Unlock() + tx, err := q.f.db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + var state string + switch err := tx.QueryRow(fmt.Sprintf("SELECT state FROM %s WHERE job_id = ?", q.jobs), id).Scan(&state); { + case err == nil: + return contract.ErrJobEnded // every record is an end + case !errors.Is(err, sql.ErrNoRows): + return err + } + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE expires_at <= ?", q.cancelled), now); err != nil { + return err + } + // REPLACE deletes an older mark of id, so the new one takes a new seq + // and reaches OnCancel again + if _, err := tx.Exec(fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, expires_at) VALUES (?, ?)", q.cancelled), + id, now+cancelTTL.Milliseconds()); err != nil { + return err + } + return tx.Commit() +} + +// AcknowledgeCancelled implements contract.CancelAcknowledgeable: Reject +// with outcome cancelled and reason as its error. A kept row +// (WithRemoveOnComplete(false)) gets status 'cancelled'. +func (q *Queue) AcknowledgeCancelled(ackID, reason string) bool { + return q.finish(ackID, contract.JobCancelled, reason, nil) +} + +// OnCancel implements contract.CancelNotifier: fn gets the job ID of each +// cancel request made after the call, through any queue object on this +// database file, until stop. Marks are polled every pollEvery; each call +// runs on its own goroutine. A mark gone by the poll (the job finished or +// was re-added) is never reported. A closed queue, or a failed first read, +// reports nothing. +func (q *Queue) OnCancel(fn func(jobID string)) (stop func()) { + // Read before the lock: a mark made after this read has a higher seq + var last sql.NullInt64 + if err := q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.cancelled)).Scan(&last); err != nil { + return func() {} // 0 would replay every live mark + } + + h := &cancelHook{fn: fn, from: last.Int64} + w := &q.w + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return func() {} + } + if len(w.hooks) == 0 { + w.lastSeq = h.from // no poll read marks for anyone since + } + if w.hooks == nil { + w.hooks = map[*cancelHook]struct{}{} + } + w.hooks[h] = struct{}{} + w.startLocked(q) + + return func() { + w.mu.Lock() + defer w.mu.Unlock() + delete(w.hooks, h) + } +} + +// cancelHook is one OnCancel callback; it gets marks with seq above from. +type cancelHook struct { + fn func(string) + from int64 +} + +// pollCancels reads the live marks newer than seq and passes each to the +// hooks live after the read whose from is below it, so a hook added during +// the read still gets its marks, and a stopped one gets none. +func (q *Queue) pollCancels(seq int64) { + rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? AND expires_at > ? ORDER BY seq LIMIT %d", + q.cancelled, maxDrops), seq, time.Now().UnixMilli()) + if err != nil { + return + } + type mark struct { + seq int64 + id string + } + var marks []mark + for rows.Next() { + var m mark + if rows.Scan(&m.seq, &m.id) == nil { + marks = append(marks, m) + } + } + if rows.Close() != nil || rows.Err() != nil || len(marks) == 0 { + return // read again from the old seq + } + + w := &q.w + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + w.lastSeq = max(w.lastSeq, marks[len(marks)-1].seq) + for h := range w.hooks { + for _, m := range marks { + if m.seq > h.from { + go h.fn(m.id) + } + } + } +} + +// anyMark reports whether a live mark exists, so a pop without marks pays +// one read; an error counts as a mark (the gated pop then reports it). +func (q *Queue) anyMark(now time.Time) bool { + var n int + if err := q.hasMark.QueryRow(now.UnixMilli()).Scan(&n); err != nil { + return true + } + return n == 1 +} + +// marked reports whether job id has a live mark; isMarked is q.isMarked +// in the caller's tx. +func marked(isMarked *sql.Stmt, id string, now time.Time) (bool, error) { + if id == "" { + return false, nil + } + var n int + err := isMarked.QueryRow(id, now.UnixMilli()).Scan(&n) + return n == 1, err +} diff --git a/commit.go b/commit.go index edb4f29..be6fed2 100644 --- a/commit.go +++ b/commit.go @@ -10,7 +10,7 @@ import ( // code (such as a driver.Valuer): a panic there would strand the round and // leave its transaction open. Convert user values before building it. type op struct { - run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit + run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit, or a tx of its own done chan result // buffered 2: one lead wake-up and the outcome } @@ -97,6 +97,7 @@ func (b *batcher) lead() { // transaction of its own, so one bad call fails alone. A round of one op runs // with a nil tx, as one autocommit statement: a database/sql Tx costs a // context and a goroutine, which a round with nothing to share does not repay. +// An op of several statements (a finish) opens its own tx then. func (f *file) commitRound(ops []*op) { f.mu.Lock() defer f.mu.Unlock() diff --git a/commit_test.go b/commit_test.go index 6b1a7e1..54fa30d 100644 --- a/commit_test.go +++ b/commit_test.go @@ -491,3 +491,53 @@ func TestGroupCommitAfterClose(t *testing.T) { } waitAll(t, &wg, 10*time.Second) } + +// Finishes that share a round write their outcomes in its transaction, +// and again each in its own when a bad call fails the round. +func TestGroupFinishOutcomes(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run(map[bool]string{false: "shared", true: "fallback"}[fail], func(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + for _, id := range []string{"a", "b", "c", "d"} { + q.Enqueue(job(id)) + } + items, acks := make([]any, 4), make([]string, 4) + if n, _, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != 4 { + t.Fatalf("got %d %v", n, err) + } + rest := []func() bool{ + func() bool { return q.Reject(acks[1], "r") }, + func() bool { return q.AcknowledgeResult(acks[2], []byte(`1`)) }, + func() bool { return q.AcknowledgeCancelled(acks[3], "c") }, + } + if fail { + rest = append(rest, func() bool { return q.Enqueue(nil) }) + } + got := oneRound(t, q.f, func() bool { return q.Acknowledge(acks[0]) }, rest) + for i, ok := range got[:4] { + if !ok { + t.Errorf("finish %d failed", i) + } + } + want := map[string]string{"a": "completed||", "b": "failed|r|", "c": "completed||1", "d": "cancelled|c|"} + rows, err := q.f.db.Query("SELECT job_id, state, COALESCE(error, ''), COALESCE(CAST(result AS TEXT), '') FROM " + q.jobs) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + n := 0 + for rows.Next() { + var id, state, e, r string + rows.Scan(&id, &state, &e, &r) + if g := state + "|" + e + "|" + r; g != want[id] { + t.Errorf("%s: got %s, want %s", id, g, want[id]) + } + n++ + } + if n != 4 { + t.Errorf("Expected 4 records, got %d", n) + } + }) + } +} diff --git a/go.mod b/go.mod index b6ff707..d59e374 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,9 @@ module github.com/goptics/sqliteq -go 1.24 +go 1.27 -require github.com/mattn/go-sqlite3 v1.14.28 +require ( + github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6 + github.com/goptics/varmq/contract v0.6.0 + github.com/mattn/go-sqlite3 v1.14.28 +) diff --git a/go.sum b/go.sum index 42e5bac..f800d0f 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,14 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6 h1:0GooGJC22Z/9WxAz0f8aVzWyPkBDJhHF8JI4oYTIW+w= +github.com/goptics/varmq v1.10.1-0.20261108161945-6bc553db29c6/go.mod h1:RLnozQPouu3sq2tHoCixCXSoCZMPjB90MNEHvlpFz+E= +github.com/goptics/varmq/contract v0.6.0 h1://rpDlzSwsqvPDS98siuPDN1LyyBLFhiYeAmB7MZfbQ= +github.com/goptics/varmq/contract v0.6.0/go.mod h1:55tOUywUsZhBgyo8oJhNmKMknETVHL2OuMrITztanXg= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/migration_test.go b/migration_test.go index bc94204..be706f8 100644 --- a/migration_test.go +++ b/migration_test.go @@ -129,6 +129,12 @@ func TestMigrateV02File(t *testing.T) { if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { t.Errorf("Round %d: indexes = %v, want %v", round, got, want) } + for _, side := range []string{name + jobsSuffix, name + cancelledSuffix, name + readdSuffix} { + var n int + if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = ?", side).Scan(&n); err != nil || n != 1 { + t.Errorf("Round %d: expected %s on the old file, got %d %v", round, side, n, err) + } + } if round == 1 { // The two processing rows are pending again after recovery diff --git a/outcome.go b/outcome.go new file mode 100644 index 0000000..643f2a3 --- /dev/null +++ b/outcome.go @@ -0,0 +1,424 @@ +package sqliteq + +import ( + "database/sql" + "encoding/json" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/goptics/varmq/contract" +) + +// Job outcome (varmq spec/redis-storage.md 1.6.0, 6.13), on SQLite. Each +// finish of a job with a Job ID writes one row to __sqliteq_jobs in +// the finish's transaction, under a new AUTOINCREMENT seq. SQLite has no +// pub/sub, so a watcher is answered by polling that table by seq (records +// newer than the last read): one goroutine per queue object, alive only +// while it has watches or OnCancel hooks. It also sweeps retention and +// expired marks. + +var ( + _ contract.OutcomeWatcher = (*Queue)(nil) + _ contract.OutcomeWatcher = (*PriorityQueue)(nil) + _ contract.ResultAcknowledgeable = (*Queue)(nil) + _ contract.ResultAcknowledgeable = (*PriorityQueue)(nil) +) + +const ( + // pollEvery is how often a queue object with watches or hooks reads + // the outcome and cancel tables. + pollEvery = 50 * time.Millisecond + // pollBatch caps the rows one cursor read returns; a poll reads again + // until it gets fewer. + pollBatch = 1000 + // sweepEvery is how often a polling queue object trims retention and + // drops expired marks. + sweepEvery = time.Minute + + // Retention (spec 5 defaults): cancelled records count as failed. + keepCompleted = 1000 + keepCompletedAge = 24 * time.Hour + keepFailed = 10000 + keepFailedAge = 7 * 24 * time.Hour + // trimEvery is how many records a queue object writes between trims. + trimEvery = 100 + // trimFloor: the count limit never drops a record younger than this, + // so a burst can't trim one before a poll reads it. + trimFloor = time.Minute + + jobsSuffix = "__sqliteq_jobs" + cancelledSuffix = "__sqliteq_cancelled" + readdSuffix = "__sqliteq_readd" +) + +// initSidecars creates the outcome and cancel tables, and the trigger that +// clears both for an id EnqueueOnce stores (spec 6.13 and 6.14 Re-add): an +// ON CONFLICT DO NOTHING insert of a repeat id fires no trigger. The caller +// holds the write mutex. +func (q *Queue) initSidecars() error { + for _, s := range []string{ + // AUTOINCREMENT: seq never reuses a deleted row's, so a seq > last + // cursor sees every new record and mark; REPLACE gives a new seq + `CREATE TABLE IF NOT EXISTS %[2]s ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL UNIQUE, + state TEXT NOT NULL, + error TEXT, + result BLOB, + finished_at INTEGER NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS %[4]s ON %[2]s (finished_at)`, + `CREATE TABLE IF NOT EXISTS %[3]s ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL UNIQUE, + expires_at INTEGER NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS %[6]s ON %[3]s (expires_at)`, + `CREATE TRIGGER IF NOT EXISTS %[5]s AFTER INSERT ON %[1]s WHEN NEW.job_id IS NOT NULL BEGIN + DELETE FROM %[2]s WHERE job_id = NEW.job_id; + DELETE FROM %[3]s WHERE job_id = NEW.job_id; + END`, + } { + query := fmt.Sprintf(s, q.table, q.jobs, q.cancelled, + quoteIdent(q.name+jobsSuffix+"_finished"), quoteIdent(q.name+readdSuffix), + quoteIdent(q.name+cancelledSuffix+"_expires")) + if _, err := q.f.db.Exec(query); err != nil { + return err + } + } + return nil +} + +// jobIDOf is a row's Job ID: its EnqueueOnce id, else the envelope "id" of +// a JSON object item (varmq's generated ids come with plain Enqueue). "" +// when it has neither: such a job has no outcome and is never cancelled. +func jobIDOf(onceID sql.NullString, data []byte) string { + if onceID.Valid { + return onceID.String + } + if len(data) == 0 || data[0] != '{' { + return "" + } + var env struct { + ID string `json:"id"` + } + if json.Unmarshal(data, &env) != nil { + return "" + } + return env.ID +} + +// record writes job id's outcome and drops its cancel mark, in tx; every +// trimEvery-th record also trims both states. +func (q *Queue) record(tx *sql.Tx, id string, state contract.JobState, reason string, result []byte, now time.Time) error { + var errText any + if state != contract.JobCompleted { + errText = reason + } + if _, err := tx.Stmt(q.putJob).Exec(id, string(state), errText, result, now.UnixMilli()); err != nil { + return err + } + if _, err := tx.Stmt(q.unmark).Exec(id); err != nil { + return err + } + // The first record of a queue object trims too: a short-lived process + // may write fewer than trimEvery + if q.records.Add(1)%trimEvery == 1 { + return q.trim(tx, now) + } + return nil +} + +// trim applies retention: per state group, records past the age limit and +// those past the newest N that are older than trimFloor. +func (q *Queue) trim(tx *sql.Tx, now time.Time) error { + for _, g := range []struct { + where string + keep int + age time.Duration + }{ + {"state = 'completed'", keepCompleted, keepCompletedAge}, + {"state <> 'completed'", keepFailed, keepFailedAge}, + } { + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE %s AND finished_at <= ?", q.jobs, g.where), + now.Add(-g.age).UnixMilli()); err != nil { + return err + } + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %[1]s WHERE finished_at <= ? AND job_id IN (SELECT job_id FROM %[1]s WHERE %[2]s ORDER BY finished_at DESC LIMIT -1 OFFSET %[3]d)", + q.jobs, g.where, g.keep), now.Add(-trimFloor).UnixMilli()); err != nil { + return err + } + } + return nil +} + +// AcknowledgeResult implements contract.ResultAcknowledgeable: Acknowledge +// that keeps result, which must be valid JSON, in the job's outcome. Not +// JSON: false, and nothing changes. +func (q *Queue) AcknowledgeResult(ackID string, result []byte) bool { + if !json.Valid(result) { + return false + } + return q.finish(ackID, contract.JobCompleted, "", result) +} + +type outcomeWatch struct { + fn func(contract.JobOutcome) + gen uint64 // w.gen at its start + done atomic.Bool // called or cancelled +} + +// watchers are a queue object's outcome watches and OnCancel hooks, and the +// state of its one poll goroutine. mu is never held across a query. +type watchers struct { + mu sync.Mutex + byID map[string]map[*outcomeWatch]struct{} + hooks map[*cancelHook]struct{} + lastSeq int64 // newest cancel mark seq a poll has read + outSeq int64 // newest outcome record seq a poll has read + gen uint64 // watches started; a read reaches only watches started before it + polling bool + closed bool + stop chan struct{} // closed by close +} + +// startLocked starts the poll goroutine if none runs; w.mu is held. +func (w *watchers) startLocked(q *Queue) { + if !w.polling { + w.polling = true + go q.poll() + } +} + +// close drops every watch and hook without calling it, and stops polling. +func (w *watchers) close() { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + w.closed = true + close(w.stop) + for _, set := range w.byID { + for o := range set { + o.done.Store(true) + } + } + w.byID, w.hooks = nil, nil +} + +// WatchOutcome implements contract.OutcomeWatcher: fn runs once, on its own +// goroutine, when job id completes, fails or is cancelled through any +// queue object on this database file, in this process or another. A job +// whose record is still kept is reported even if it ended before the call. +// It is polled every pollEvery. An empty id, or a closed queue, watches +// nothing. Close drops watches without calling fn. +func (q *Queue) WatchOutcome(id string, fn func(contract.JobOutcome)) (cancel func()) { + o := &outcomeWatch{fn: fn} + w := &q.w + w.mu.Lock() + if id == "" || w.closed { + w.mu.Unlock() + return func() {} + } + if len(w.byID) == 0 { + // No poll moved the cursor while nothing was watched: start it at + // the newest record. Older ones are this watch's catchUp below. + w.mu.Unlock() + var last sql.NullInt64 + err := q.f.db.QueryRow(fmt.Sprintf("SELECT MAX(seq) FROM %s", q.jobs)).Scan(&last) + w.mu.Lock() + if w.closed { + w.mu.Unlock() + return func() {} + } + if err == nil && len(w.byID) == 0 { + w.outSeq = last.Int64 + } + } + w.gen++ + o.gen = w.gen + if w.byID == nil { + w.byID = map[string]map[*outcomeWatch]struct{}{} + } + if w.byID[id] == nil { + w.byID[id] = map[*outcomeWatch]struct{}{} + } + w.byID[id][o] = struct{}{} + w.startLocked(q) + w.mu.Unlock() + + go q.catchUp([]string{id}, o.gen) + + return func() { + if o.done.CompareAndSwap(false, true) { + w.mu.Lock() + defer w.mu.Unlock() + if set := w.byID[id]; set != nil { + delete(set, o) + if len(set) == 0 { + delete(w.byID, id) + } + } + } + } +} + +// poll reads new outcome records and cancel marks every pollEvery, and +// sweeps every sweepEvery, until Close or until nothing is watched. +func (q *Queue) poll() { + t := time.NewTicker(pollEvery) + defer t.Stop() + w := &q.w + swept := time.Now() + for { + select { + case <-w.stop: + return + case <-t.C: + } + + w.mu.Lock() + if len(w.byID) == 0 && len(w.hooks) == 0 { + w.polling = false + w.mu.Unlock() + return + } + watching, hooks := len(w.byID) > 0, len(w.hooks) > 0 + outSeq, seq, gen := w.outSeq, w.lastSeq, w.gen + w.mu.Unlock() + + if watching { + q.pollOutcomes(outSeq, gen) + } + if hooks { + q.pollCancels(seq) + } + if time.Since(swept) >= sweepEvery { + swept = time.Now() + q.sweep() + } + } +} + +// pollOutcomes reads the records newer than seq, in pollBatch reads, and +// delivers the watched ones to watches up to gen. +func (q *Queue) pollOutcomes(seq int64, gen uint64) { + for { + rows, err := q.f.db.Query(fmt.Sprintf("SELECT seq, job_id FROM %s WHERE seq > ? ORDER BY seq LIMIT %d", q.jobs, pollBatch), seq) + if err != nil { + return + } + var ids []string + for rows.Next() { + var id string + if rows.Scan(&seq, &id) == nil { + ids = append(ids, id) + } + } + if rows.Close() != nil || rows.Err() != nil || len(ids) == 0 { + return // read again from the old seq + } + + w := &q.w + w.mu.Lock() + w.outSeq = max(w.outSeq, seq) + watched := ids[:0] + for _, id := range ids { + if w.byID[id] != nil { + watched = append(watched, id) + } + } + w.mu.Unlock() + if len(watched) > 0 { + q.catchUp(watched, gen) + } + if len(ids) < pollBatch { + return + } + } +} + +// sweep applies retention and drops expired cancel marks (spec 5, 6.14 +// Prune), so an idle queue ages out too. +func (q *Queue) sweep() { + q.f.mu.Lock() + defer q.f.mu.Unlock() + tx, err := q.f.db.Begin() + if err != nil { + return + } + defer tx.Rollback() + now := time.Now() + if q.trim(tx, now) != nil { + return + } + if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE expires_at <= ?", q.cancelled), now.UnixMilli()); err != nil { + return + } + tx.Commit() +} + +// catchUp delivers the ids whose outcome record exists to their watches +// up to gen, those started before the read. A failed read is retried by +// the next poll. +func (q *Queue) catchUp(ids []string, gen uint64) { + args := make([]any, len(ids)) + for i, id := range ids { + args[i] = id + } + rows, err := q.f.db.Query(fmt.Sprintf("SELECT job_id, state, error, result FROM %s WHERE job_id IN (%s)", + q.jobs, strings.Repeat("?, ", len(ids)-1)+"?"), args...) + if err != nil { + return + } + var outs []contract.JobOutcome + for rows.Next() { + var ( + id, state string + errText sql.NullString + result []byte + ) + if rows.Scan(&id, &state, &errText, &result) != nil { + continue + } + outs = append(outs, contract.JobOutcome{ID: id, State: contract.JobState(state), Error: errText.String, Result: result}) + } + rows.Close() + + for _, out := range outs { + q.deliver(out, gen) + } +} + +// deliver ends the watches of out.ID started up to gen: each fn runs once, +// on its own goroutine, so a slow fn holds up no poll. A watch started +// after the read stays: the record read may be an earlier run's, which a +// re-add since deleted (spec 6.13 Re-add). +func (q *Queue) deliver(out contract.JobOutcome, gen uint64) { + w := &q.w + w.mu.Lock() + set := w.byID[out.ID] + var ended []*outcomeWatch + for o := range set { + if o.gen <= gen { + ended = append(ended, o) + delete(set, o) + } + } + if len(set) == 0 { + delete(w.byID, out.ID) + } + w.mu.Unlock() + + for _, o := range ended { + go func() { + if o.done.CompareAndSwap(false, true) { + o.fn(out) + } + }() + } +} diff --git a/priority_queue_test.go b/priority_queue_test.go index 39b2ed7..a7cadcf 100644 --- a/priority_queue_test.go +++ b/priority_queue_test.go @@ -1,10 +1,13 @@ package sqliteq import ( + "errors" "fmt" "strconv" "testing" + "github.com/goptics/varmq/contract" + _ "github.com/mattn/go-sqlite3" ) @@ -653,3 +656,44 @@ func TestBatchPriorityOrder(t *testing.T) { } } } + +// A priority queue reports outcomes, drops marked waiting jobs in priority +// order, and re-add clears the mark, as Queue does. +func TestPriorityOutcomeAndCancel(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("pq") + if err != nil { + t.Fatal(err) + } + other := otherProcess(t, pq.Queue) + + pq.Enqueue(job("low"), 5) + pq.Enqueue(job("high"), 1) + pq.EnqueueOnce(job("mid"), 3, "mid") + if err := other.CancelJob("high"); err != nil { + t.Fatal(err) + } + chHigh, chMid := watch(other, "high"), watch(other, "mid") + + if v, ack := pop(t, pq.Queue); v != string(job("mid")) { + t.Fatalf("got %s, want mid", v) + } else if !pq.AcknowledgeResult(ack, []byte(`7`)) { + t.Fatal("AcknowledgeResult failed") + } + if got := waitOutcome(t, chHigh); got.State != contract.JobCancelled { + t.Errorf("high: got %+v", got) + } + if got := waitOutcome(t, chMid); got.State != contract.JobCompleted || string(got.Result) != "7" { + t.Errorf("mid: got %+v", got) + } + if err := pq.CancelJob("mid"); !errors.Is(err, contract.ErrJobEnded) { + t.Errorf("got %v, want ErrJobEnded", err) + } + + pq.CancelJob("again") + pq.EnqueueOnce(job("again"), 0, "again") + if v, _ := pop(t, pq.Queue); v != string(job("again")) { + t.Errorf("Expected the re-added job first, got %s", v) + } +} diff --git a/queue.go b/queue.go index 7d41be6..be1fbdd 100644 --- a/queue.go +++ b/queue.go @@ -4,6 +4,7 @@ import ( "cmp" "database/sql" "database/sql/driver" + "errors" "fmt" "slices" "strconv" @@ -11,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/goptics/varmq/contract" _ "github.com/mattn/go-sqlite3" ) @@ -26,8 +28,13 @@ type Queue struct { removeOnComplete bool closed atomic.Bool + jobs, cancelled string // quoted outcome and cancel mark tables + w watchers // outcome watches and OnCancel hooks + records atomic.Int64 // outcome records written, for trims + // Prepared once per handle; they close with the file's pool - insert, claim, pop, ack, reject, count, countCapped, values *sql.Stmt + insert, claim, pop, fin, count, countCapped, values *sql.Stmt + putJob, unmark, hasMark, isMarked *sql.Stmt } // claimed is one row a claim moved to processing @@ -35,6 +42,7 @@ type claimed struct { id int64 priority int data []byte + once sql.NullString // job_id } // newQueue creates a SQLite-based queue on its table: the table itself, the @@ -48,7 +56,10 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro priority: priority, order: "id", removeOnComplete: true, // Default to removing completed items + jobs: quoteIdent(name + jobsSuffix), + cancelled: quoteIdent(name + cancelledSuffix), } + q.w.stop = make(chan struct{}) if priority { q.order = "priority, id" } @@ -86,6 +97,9 @@ func (q *Queue) initSchema() error { if err := q.addJobID(); err != nil { return fmt.Errorf("failed to add job_id: %w", err) } + if err := q.initSidecars(); err != nil { + return fmt.Errorf("failed to create outcome tables: %w", err) + } return nil } @@ -198,18 +212,16 @@ func (q *Queue) prepare() (err error) { } }() - cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data" + cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data, job_id" if q.priority { - cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data" + cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data, job_id" } pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order) - ack := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing'", q.table) - reject := ack + // fin ends a processing row: deleted, or kept with its end as status + fin := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table) if !q.removeOnComplete { - finish := "UPDATE %s SET status = '%s', ack = 1, updated_at = ? WHERE id = ? AND status = 'processing'" - ack = fmt.Sprintf(finish, q.table, "completed") - reject = fmt.Sprintf(finish, q.table, "failed") + fin = fmt.Sprintf("UPDATE %s SET status = ?, ack = 1, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table) } for _, s := range []struct { @@ -220,9 +232,12 @@ func (q *Queue) prepare() (err error) { // other failure still errors; it must repeat the index's WHERE {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING", q.table, cols, vals)}, {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, - {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)}, - {&q.ack, ack}, - {&q.reject, reject}, + {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data, job_id", q.table, pending)}, + {&q.fin, fin}, + {&q.putJob, fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, state, error, result, finished_at) VALUES (?, ?, ?, ?, ?)", q.jobs)}, + {&q.unmark, fmt.Sprintf("DELETE FROM %s WHERE job_id = ?", q.cancelled)}, + {&q.hasMark, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE expires_at > ?)", q.cancelled)}, + {&q.isMarked, fmt.Sprintf("SELECT EXISTS (SELECT 1 FROM %s WHERE job_id = ? AND expires_at > ?)", q.cancelled)}, {&q.count, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", q.table)}, {&q.countCapped, fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE status = 'pending' LIMIT %d)", q.table, remainingCap)}, {&q.values, fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)}, @@ -237,7 +252,7 @@ func (q *Queue) prepare() (err error) { // closeStmts closes the statements prepared so far func (q *Queue) closeStmts() { - for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.ack, q.reject, q.count, q.countCapped, q.values} { + for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.fin, q.count, q.countCapped, q.values, q.putJob, q.unmark, q.hasMark, q.isMarked} { if s != nil { s.Close() } @@ -349,12 +364,12 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { // claimN moves up to n pending rows to processing in one statement and // returns them in queue order -func (q *Queue) claimN(n int) ([]claimed, error) { +func (q *Queue) claimN(n int) ([]claimed, bool, error) { q.f.mu.Lock() - out, err := q.claimRows(n) + out, capped, err := q.claimGated(n) q.f.mu.Unlock() if err != nil { - return nil, err + return nil, false, err } // RETURNING order is unspecified @@ -364,13 +379,61 @@ func (q *Queue) claimN(n int) ([]claimed, error) { } return cmp.Compare(a.id, b.id) }) - return out, nil + return out, capped, nil +} + +// claimGated claims up to n rows; with a live cancel mark on the file it +// runs in one transaction that drops each marked row it claims (outcome +// cancelled) and claims again, at most maxDrops drops; capped reports that +// it stopped there. The caller holds the write mutex. +func (q *Queue) claimGated(n int) (out []claimed, capped bool, err error) { + now := time.Now() + if !q.anyMark(now) { + out, err = q.claimRows(q.claim, n) + return out, false, err + } + + tx, err := q.f.db.Begin() + if err != nil { + return nil, false, err + } + defer tx.Rollback() + + claim, isMarked := tx.Stmt(q.claim), tx.Stmt(q.isMarked) + out = make([]claimed, 0, n) + drops := 0 + for len(out) < n && drops < maxDrops { + // No more than the drops left: a batch never passes the cap + rows, err := q.claimRows(claim, min(n-len(out), maxDrops-drops)) + if err != nil { + return nil, false, err + } + if len(rows) == 0 { + break + } + for _, c := range rows { + id := jobIDOf(c.once, c.data) + drop, err := marked(isMarked, id, now) + if err != nil { + return nil, false, err + } + if !drop { + out = append(out, c) + continue + } + if _, err := q.end(tx, c.id, id, contract.JobCancelled, "cancelled", nil, now); err != nil { + return nil, false, err + } + drops++ + } + } + return out, len(out) < n && drops >= maxDrops, tx.Commit() } -// claimRows runs the claim statement and reads every row it returns; the +// claimRows runs a claim statement and reads every row it returns; the // caller holds the write mutex -func (q *Queue) claimRows(n int) ([]claimed, error) { - rows, err := q.claim.Query(time.Now().UTC(), n) +func (q *Queue) claimRows(claim *sql.Stmt, n int) ([]claimed, error) { + rows, err := claim.Query(time.Now().UTC(), n) if err != nil { return nil, err } @@ -380,9 +443,9 @@ func (q *Queue) claimRows(n int) ([]claimed, error) { for rows.Next() { var c claimed if q.priority { - err = rows.Scan(&c.id, &c.priority, &c.data) + err = rows.Scan(&c.id, &c.priority, &c.data, &c.once) } else { - err = rows.Scan(&c.id, &c.data) + err = rows.Scan(&c.id, &c.data, &c.once) } if err != nil { return nil, err @@ -403,25 +466,57 @@ func (q *Queue) Dequeue() (any, bool) { q.f.mu.Lock() defer q.f.mu.Unlock() - // Deferred last, so the rows close (and the statement ends) before unlocking - rows, err := q.pop.Query() - if err != nil { - return nil, false + now := time.Now() + if !q.anyMark(now) { + data, _, err := popRow(q.pop) + return data, err == nil } - defer rows.Close() - - var data []byte - found := false - for rows.Next() { - if err := rows.Scan(&data); err != nil { - return nil, false + for { + data, capped, err := q.popGated(now) + if err != nil || data != nil || !capped { + return data, err == nil && data != nil } - found = true } - if rows.Err() != nil || !found { - return nil, false +} + +// popGated pops in one transaction, dropping each marked item it pops +// (outcome cancelled), at most maxDrops; capped reports it stopped there +// with nothing. The caller holds the write mutex. +func (q *Queue) popGated(now time.Time) (data []byte, capped bool, err error) { + tx, err := q.f.db.Begin() + if err != nil { + return nil, false, err + } + defer tx.Rollback() + pop, isMarked := tx.Stmt(q.pop), tx.Stmt(q.isMarked) + for range maxDrops { + data, once, err := popRow(pop) + if errors.Is(err, sql.ErrNoRows) { + return nil, false, tx.Commit() // keep the drops + } + if err != nil { + return nil, false, err + } + id := jobIDOf(once, data) + drop, err := marked(isMarked, id, now) + if err != nil { + return nil, false, err + } + if !drop { + return data, false, tx.Commit() + } + if err := q.record(tx, id, contract.JobCancelled, "cancelled", nil, now); err != nil { + return nil, false, err + } } - return data, true + return nil, true, tx.Commit() +} + +// popRow runs a pop statement: the deleted row's data and job_id, or +// sql.ErrNoRows on an empty queue +func popRow(pop *sql.Stmt) (data []byte, once sql.NullString, err error) { + err = pop.QueryRow().Scan(&data, &once) + return data, once, err } // DequeueWithAckId moves the next item to processing and returns it with its @@ -431,11 +526,17 @@ func (q *Queue) DequeueWithAckId() (any, bool, string) { return nil, false, "" } - c, err := q.claimN(1) - if err != nil || len(c) == 0 { - return nil, false, "" + // A pop stopped at the drop cap has nothing yet: pop again, it can't + // report items left + for { + c, capped, err := q.claimN(1) + if err != nil || len(c) == 0 && !capped { + return nil, false, "" + } + if len(c) > 0 { + return c[0].data, true, strconv.FormatInt(c[0].id, 10) + } } - return c[0].data, true, strconv.FormatInt(c[0].id, 10) } // DequeueBatchWithAckIds implements varmq.IBatchAcknowledgeable: it moves up @@ -449,12 +550,17 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini return 0, 0, nil } - c, err := q.claimN(want) + c, capped, err := q.claimN(want) if err != nil { return 0, 0, err } + // At the drop cap items may wait still: report one, so the caller pops again + left := 0 + if capped { + left = 1 + } if len(c) == 0 { - return 0, 0, nil + return 0, left, nil } for i, row := range c { items[i], ackIDs[i] = row.data, strconv.FormatInt(row.id, 10) @@ -465,52 +571,83 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini if err := q.countCapped.QueryRow().Scan(&remaining); err != nil { return len(c), 1, nil } - return len(c), remaining, nil + return len(c), max(remaining, left), nil } // Acknowledge marks a processing item as completed, deleting it unless // WithRemoveOnComplete(false) is set, in the file's next group commit round // Returns true only if exactly one processing row had this ack ID -func (q *Queue) Acknowledge(ackID string) bool { return q.finish(ackID, q.ack) } +func (q *Queue) Acknowledge(ackID string) bool { + return q.finish(ackID, contract.JobCompleted, "", nil) +} // Reject marks a processing item as failed for good: it is not requeued. // The row is deleted, or with WithRemoveOnComplete(false) kept with status -// 'failed'. Same group commit and result as Acknowledge. -// ponytail: reason is not stored (no column); add one when something reads it. -func (q *Queue) Reject(ackID string, reason string) bool { return q.finish(ackID, q.reject) } +// 'failed'. reason is the error of the job's outcome. Same group commit and +// result as Acknowledge. +func (q *Queue) Reject(ackID string, reason string) bool { + return q.finish(ackID, contract.JobFailed, reason, nil) +} -// finish runs stmt (ack or reject) for the row of ackID. -func (q *Queue) finish(ackID string, stmt *sql.Stmt) bool { +// finish ends the processing row of ackID with state in the file's next +// group commit round. A round of one runs in its own transaction: the row +// and its outcome record commit together. +func (q *Queue) finish(ackID string, state contract.JobState, reason string, result []byte) bool { id, ok := parseAckID(ackID) if !ok { return false } - now := time.Now().UTC() - return q.f.b.do(func(tx *sql.Tx) (bool, error) { - var ( - result sql.Result - err error - ) - s := stmt - if tx != nil { - s = tx.Stmt(s) - } - if q.removeOnComplete { - result, err = s.Exec(id) - } else { - result, err = s.Exec(now, id) + now := time.Now() + return q.f.b.do(func(tx *sql.Tx) (ok bool, err error) { + if tx == nil { + if tx, err = q.f.db.Begin(); err != nil { + return false, err + } + defer func() { + if err == nil { + err = tx.Commit() + } else { + tx.Rollback() + } + ok = ok && err == nil + }() } - if err != nil { - return false, err - } - // No row is an outcome, not an error: it must not fail the round - n, err := result.RowsAffected() - return err == nil && n == 1, err + return q.end(tx, id, "", state, reason, result, now) }) } +// end finishes processing row id with state, in tx: the row is deleted or +// kept with state as status, and a job with a Job ID gets its outcome +// record. jobID is the row's, when the caller knows it. Reports whether +// the row was processing. +func (q *Queue) end(tx *sql.Tx, id int64, jobID string, state contract.JobState, reason string, result []byte, now time.Time) (bool, error) { + var row *sql.Row + if q.removeOnComplete { + row = tx.Stmt(q.fin).QueryRow(id) + } else { + row = tx.Stmt(q.fin).QueryRow(string(state), now.UTC(), id) + } + var ( + once sql.NullString + data []byte + ) + switch err := row.Scan(&once, &data); { + case errors.Is(err, sql.ErrNoRows): + return false, nil + case err != nil: + return false, err + } + if jobID == "" { + jobID = jobIDOf(once, data) + } + if jobID == "" { + return true, nil + } + return true, q.record(tx, jobID, state, reason, result, now) +} + // Len returns the number of pending items in the queue func (q *Queue) Len() int { var count int @@ -543,17 +680,30 @@ func (q *Queue) Values() []any { return items } -// Purge removes all items from the queue +// Purge removes all items from the queue, with its outcome records and +// cancel marks (spec 6.6, 6.14) func (q *Queue) Purge() { q.f.mu.Lock() defer q.f.mu.Unlock() - q.f.db.Exec(fmt.Sprintf("DELETE FROM %s", q.table)) + tx, err := q.f.db.Begin() + if err != nil { + return + } + defer tx.Rollback() + for _, t := range []string{q.table, q.jobs, q.cancelled} { + if _, err := tx.Exec("DELETE FROM " + t); err != nil { + return + } + } + tx.Commit() } -// Close closes the queue and its database connection +// Close closes the queue and drops its outcome watches and OnCancel hooks +// without calling them. The database file stays open for the Queues. func (q *Queue) Close() error { q.closed.Store(true) + q.w.close() return nil } diff --git a/queue_test.go b/queue_test.go index bf66ab0..6f55614 100644 --- a/queue_test.go +++ b/queue_test.go @@ -1,14 +1,20 @@ package sqliteq import ( + "context" "database/sql" + "errors" "fmt" "path/filepath" + "slices" "strconv" "sync" "sync/atomic" "testing" "time" + + "github.com/goptics/varmq" + "github.com/goptics/varmq/contract" ) func TestSQLiteQueue(t *testing.T) { @@ -844,6 +850,9 @@ func TestWritesTakeFileMutex(t *testing.T) { {"DequeueWithAckId", func() { q.DequeueWithAckId() }}, {"DequeueBatchWithAckIds", func() { q.DequeueBatchWithAckIds(make([]any, 2), make([]string, 2)) }}, {"Acknowledge", func() { q.Acknowledge(ack1) }}, + {"CancelJob", func() { q.CancelJob("x") }}, + {"gated DequeueWithAckId", func() { q.DequeueWithAckId() }}, + {"gated Dequeue", func() { q.Dequeue() }}, {"requeueNoAck", func() { q.requeueNoAck() }}, {"Purge", func() { q.Purge() }}, } @@ -894,3 +903,777 @@ func TestWritesTakeFileMutex(t *testing.T) { }) } } + +// otherProcess opens q's table again on a pool of its own, as a second +// process on the same database file would. +func otherProcess(t *testing.T, q *Queue) *Queue { + t.Helper() + db, err := sql.Open(driverName(q.f.pages), dsn(q.f.key)) + if err != nil { + t.Fatal(err) + } + f := &file{key: q.f.key, db: db, pages: q.f.pages, refs: 1, handles: map[string]*handle{}, recovered: map[string]bool{}} + f.b = &batcher{commit: f.commitRound} + q2, err := newQueue(f, q.name, q.priority, WithRemoveOnComplete(q.removeOnComplete)) + if err != nil { + db.Close() + t.Fatal(err) + } + t.Cleanup(func() { + q2.Close() + db.Close() + }) + return q2 +} + +// job is a varmq-style envelope with Job ID id. +func job(id string) []byte { return []byte(`{"id":"` + id + `","data":1}`) } + +// watch starts a watch of id on q; the channel gets its one outcome. +func watch(q *Queue, id string) <-chan contract.JobOutcome { + ch := make(chan contract.JobOutcome, 2) + q.WatchOutcome(id, func(o contract.JobOutcome) { ch <- o }) + return ch +} + +func waitOutcome(t *testing.T, ch <-chan contract.JobOutcome) contract.JobOutcome { + t.Helper() + select { + case o := <-ch: + return o + case <-time.After(5 * time.Second): + t.Fatal("no outcome") + return contract.JobOutcome{} + } +} + +func noOutcome(t *testing.T, ch <-chan contract.JobOutcome) { + t.Helper() + select { + case o := <-ch: + t.Fatalf("unexpected outcome %+v", o) + case <-time.After(4 * pollEvery): + } +} + +// pop hands out the next item and its ack id, failing on none. +func pop(t *testing.T, q *Queue) (string, string) { + t.Helper() + v, ok, ack := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId found nothing") + } + return string(v.([]byte)), ack +} + +// A watch reports each way a job ends: Acknowledge, Reject with its reason, +// AcknowledgeResult with its JSON; a job without a Job ID writes nothing. +func TestWatchOutcome(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + cases := []struct { + id string + finish func(ack string) bool + want contract.JobOutcome + }{ + {"ack", q.Acknowledge, contract.JobOutcome{ID: "ack", State: contract.JobCompleted}}, + {"rej", func(a string) bool { return q.Reject(a, "boom") }, contract.JobOutcome{ID: "rej", State: contract.JobFailed, Error: "boom"}}, + {"res", func(a string) bool { return q.AcknowledgeResult(a, []byte(`{"n":42}`)) }, + contract.JobOutcome{ID: "res", State: contract.JobCompleted, Result: []byte(`{"n":42}`)}}, + } + for _, c := range cases { + ch := watch(q, c.id) + q.Enqueue(job(c.id)) + _, ack := pop(t, q) + noOutcome(t, ch) + if !c.finish(ack) { + t.Fatalf("%s: finish failed", c.id) + } + got := waitOutcome(t, ch) + if got.ID != c.want.ID || got.State != c.want.State || got.Error != c.want.Error || string(got.Result) != string(c.want.Result) { + t.Errorf("%s: got %+v, want %+v", c.id, got, c.want) + } + noOutcome(t, ch) // once + } + + q.Enqueue([]byte("plain")) + _, ack := pop(t, q) + q.Acknowledge(ack) + if n := countJobs(t, q); n != 3 { + t.Errorf("Expected 3 outcome records, got %d", n) + } +} + +func countJobs(t *testing.T, q *Queue) int { + t.Helper() + var n int + if err := q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.jobs).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +// AcknowledgeResult with invalid JSON fails and changes nothing. +func TestAcknowledgeResultInvalid(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.Enqueue(job("a")) + _, ack := pop(t, q) + if q.AcknowledgeResult(ack, []byte(`{nope`)) { + t.Fatal("Expected invalid JSON to fail") + } + if n := countRows(t, q.f.db, "status = 'processing'"); n != 1 { + t.Errorf("Expected the row still processing, got %d", n) + } + if !q.Acknowledge(ack) { + t.Error("Expected Acknowledge to finish it") + } +} + +// A job that ended before the watch is reported from its record; cancel +// stops a watch; a closed queue watches nothing. +func TestWatchEndedBeforeAndCancel(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.Enqueue(job("early")) + _, ack := pop(t, q) + q.Reject(ack, "late watcher") + if got := waitOutcome(t, watch(q, "early")); got.State != contract.JobFailed || got.Error != "late watcher" { + t.Errorf("got %+v", got) + } + + ch := make(chan contract.JobOutcome, 1) + stop := q.WatchOutcome("never", func(o contract.JobOutcome) { ch <- o }) + stop() + stop() + q.Enqueue(job("never")) + _, ack = pop(t, q) + q.Acknowledge(ack) + noOutcome(t, ch) + + q.Close() + q.WatchOutcome("early", func(o contract.JobOutcome) { ch <- o }) + noOutcome(t, ch) + if err := q.CancelJob("x"); !errors.Is(err, errClosed) { + t.Errorf("CancelJob on a closed queue: got %v, want errClosed", err) + } +} + +// Two queue objects on one file, as two processes: each learns of jobs the +// other finishes, and a cancel reaches the other's pops and OnCancel. +func TestOutcomeAcrossProcesses(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q2 := otherProcess(t, q) + + ch := watch(q, "x") + q.Enqueue(job("x")) + _, ack := pop(t, q2) + q2.Acknowledge(ack) + if got := waitOutcome(t, ch); got.ID != "x" || got.State != contract.JobCompleted { + t.Errorf("got %+v", got) + } + + ids := make(chan string, 4) + defer q2.OnCancel(func(id string) { ids <- id })() + q2.Enqueue(job("run")) + _, ack = pop(t, q2) + ch = watch(q2, "run") + if err := q.CancelJob("run"); err != nil { + t.Fatal(err) + } + select { + case id := <-ids: + if id != "run" { + t.Errorf("OnCancel got %q", id) + } + case <-time.After(5 * time.Second): + t.Fatal("OnCancel did not fire") + } + if !q2.AcknowledgeCancelled(ack, "stopped") { + t.Fatal("AcknowledgeCancelled failed") + } + if got := waitOutcome(t, ch); got.State != contract.JobCancelled || got.Error != "stopped" { + t.Errorf("got %+v", got) + } + var marks int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks) + if marks != 0 { + t.Errorf("Expected the finish to clear the mark, %d left", marks) + } + if err := q.CancelJob("run"); !errors.Is(err, contract.ErrJobEnded) { + t.Errorf("CancelJob of an ended job: got %v, want ErrJobEnded", err) + } +} + +// A marked waiting job is never handed out, by any pop: it is dropped with +// outcome cancelled and the pop goes on with the next item. +func TestCancelWaiting(t *testing.T) { + q, qs := openQueue(t, WithRemoveOnComplete(false)) + defer qs.Close() + + for _, id := range []string{"a", "b", "c", "d", "e"} { + q.Enqueue(job(id)) + } + q.EnqueueOnce([]byte("raw"), "f") // a Job ID from EnqueueOnce, not JSON + q.Enqueue([]byte("g")) + chA, chF := watch(q, "a"), watch(q, "f") + for _, id := range []string{"a", "c", "d", "f"} { + if err := q.CancelJob(id); err != nil { + t.Fatal(err) + } + } + + if v, _ := pop(t, q); v != string(job("b")) { + t.Errorf("DequeueWithAckId: got %s, want b", v) + } + items, acks := make([]any, 2), make([]string, 2) + n, _, err := q.DequeueBatchWithAckIds(items, acks) + if err != nil || n != 2 || string(items[0].([]byte)) != string(job("e")) || string(items[1].([]byte)) != "g" { + t.Errorf("DequeueBatchWithAckIds: got %d %v %v", n, err, items[:n]) + } + if got := waitOutcome(t, chA); got.State != contract.JobCancelled || got.Error != "cancelled" { + t.Errorf("got %+v", got) + } + if got := waitOutcome(t, chF); got.State != contract.JobCancelled { + t.Errorf("got %+v", got) + } + if n := countRows(t, q.f.db, "status = 'cancelled'"); n != 4 { + t.Errorf("Expected 4 kept cancelled rows, got %d", n) + } + if _, ok, _ := q.DequeueWithAckId(); ok { + t.Error("Expected nothing left") + } + + // Dequeue without ack drops too + q.Enqueue(job("h")) + q.Enqueue(job("i")) + q.CancelJob("h") + if v, ok := q.Dequeue(); !ok || string(v.([]byte)) != string(job("i")) { + t.Errorf("Dequeue: got %v %v, want i", v, ok) + } + if got := waitOutcome(t, watch(q, "h")); got.State != contract.JobCancelled { + t.Errorf("got %+v", got) + } +} + +// EnqueueOnce that stores an id drops its old outcome and its cancel mark +// (spec 6.13, 6.14 Re-add); a dedupe hit changes neither. +func TestReaddClearsOutcomeAndMark(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + q.EnqueueOnce(job("r"), "r") + _, ack := pop(t, q) + q.Acknowledge(ack) + if countJobs(t, q) != 1 { + t.Fatal("Expected a record") + } + q.EnqueueOnce(job("r"), "r") + if countJobs(t, q) != 0 { + t.Error("Expected the re-add to drop the old record") + } + ch := watch(q, "r") + noOutcome(t, ch) // the earlier run is not reported + + if err := q.CancelJob("r"); err != nil { + t.Fatal(err) + } + q.EnqueueOnce(job("r"), "r") // dedupe hit: the row is still there + if _, ok, _ := q.DequeueWithAckId(); ok { + t.Fatal("Expected the marked job dropped") + } + if got := waitOutcome(t, ch); got.State != contract.JobCancelled { + t.Errorf("got %+v", got) + } + + q.CancelJob("s") // unknown id: marked + q.EnqueueOnce(job("s"), "s") + if v, _ := pop(t, q); v != string(job("s")) { + t.Errorf("Expected the re-added job handed out, got %s", v) + } +} + +// Purge drops outcome records and cancel marks with the items. +func TestPurgeClearsOutcomesAndMarks(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.Enqueue(job("done")) + _, ack := pop(t, q) + q.Acknowledge(ack) + q.CancelJob("later") + q.Purge() + var marks int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks) + if countJobs(t, q) != 0 || marks != 0 { + t.Errorf("Expected Purge to clear records and marks, got %d and %d", countJobs(t, q), marks) + } + q.Enqueue(job("later")) + if v, _ := pop(t, q); v != string(job("later")) { + t.Errorf("Expected the purged mark gone, got %s", v) + } +} + +// OnCancel reports marks made after it, not before; stop ends it; an +// expired mark neither reaches hooks nor drops items. +func TestOnCancel(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.CancelJob("old") + ids := make(chan string, 4) + stop := q.OnCancel(func(id string) { ids <- id }) + q.CancelJob("new") + select { + case id := <-ids: + if id != "new" { + t.Errorf("got %q, want new", id) + } + case <-time.After(5 * time.Second): + t.Fatal("OnCancel did not fire") + } + stop() + q.CancelJob("after") + select { + case id := <-ids: + t.Errorf("got %q after stop", id) + case <-time.After(4 * pollEvery): + } + + q.f.db.Exec("UPDATE "+q.cancelled+" SET expires_at = ? WHERE job_id = 'old'", time.Now().Add(-time.Second).UnixMilli()) + q.Enqueue(job("old")) + if v, _ := pop(t, q); v != string(job("old")) { + t.Errorf("Expected an expired mark to drop nothing, got %s", v) + } + if err := q.CancelJob(""); err == nil { + t.Error("Expected an empty id to fail") + } +} + +// Retention trims each state group by age and count; the count limit +// spares records younger than trimFloor; a queue object's first record +// trims. +func TestOutcomeRetention(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + now := time.Now() + insert := func(tx *sql.Tx, prefix string, n int, at time.Time) { + for i := range n { + if _, err := tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES (?, 'completed', ?)", prefix+strconv.Itoa(i), at.UnixMilli()+int64(i)); err != nil { + t.Fatal(err) + } + } + } + + tx, _ := q.f.db.Begin() + insert(tx, "fresh", keepCompleted+5, now) + if err := q.trim(tx, now); err != nil { + t.Fatal(err) + } + tx.Commit() + if n := countJobs(t, q); n != keepCompleted+5 { + t.Fatalf("Expected records younger than trimFloor kept, got %d", n) + } + q.f.db.Exec("DELETE FROM " + q.jobs) + + tx, _ = q.f.db.Begin() + insert(tx, "c", keepCompleted+5, now.Add(-2*trimFloor)) + tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", now.Add(-keepFailedAge).UnixMilli()) + tx.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'cancelled', ?)", now.UnixMilli()) + if err := q.trim(tx, now); err != nil { + t.Fatal(err) + } + tx.Commit() + if n := countJobs(t, q); n != keepCompleted+1 { + t.Errorf("Expected %d records, got %d", keepCompleted+1, n) + } + var oldest string + q.f.db.QueryRow("SELECT job_id FROM " + q.jobs + " WHERE state = 'completed' ORDER BY finished_at LIMIT 1").Scan(&oldest) + if oldest != "c5" { + t.Errorf("Expected the 5 oldest completed trimmed, oldest left %s", oldest) + } + + q2 := otherProcess(t, q) // records counter at 0 + q2.f.db.Exec("UPDATE "+q.jobs+" SET finished_at = ? WHERE job_id = 'c5'", now.Add(-keepCompletedAge).UnixMilli()) + q2.Enqueue(job("one")) + _, ack := pop(t, q2) + q2.Acknowledge(ack) + var n int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.jobs + " WHERE job_id = 'c5'").Scan(&n) + if n != 0 { + t.Error("Expected a queue object's first record to trim") + } +} + +// varmq end to end: a result job added on one queue object runs on a +// worker bound to another (another process), and its handle gets the +// result; Cancel stops a running job and drops a waiting one. +func TestVarmqRemoteResultAndCancel(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q2 := otherProcess(t, q) + + started := make(chan struct{}) + ran := make(chan int, 8) + w := varmq.NewResultWorker(func(j varmq.Job[int]) (int, error) { + ran <- j.Data() + if j.Data() < 0 { + close(started) + <-j.Context().Done() + return 0, j.Context().Err() + } + return j.Data() * 2, nil + }) + defer w.Stop() + + p := varmq.NewResultQueue[int, int](q) + waiting, ok := p.Add(7) // no worker yet + if !ok { + t.Fatal("Add failed") + } + if err := waiting.Cancel(); err != nil { + t.Fatal(err) + } + w.WithQueue(q2) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + j, _ := p.Add(21) + if err := j.WaitContext(ctx); err != nil { + t.Fatal(err) + } + if r, err := j.Result(); err != nil || r != 42 { + t.Errorf("Result: got %d %v, want 42", r, err) + } + if err := waiting.WaitContext(ctx); err != nil { + t.Fatal(err) + } + if _, err := waiting.Result(); !errors.Is(err, varmq.ErrJobCancelled) { + t.Errorf("waiting job: got %v, want ErrJobCancelled", err) + } + + running, _ := p.Add(-1) + select { + case <-started: + case <-ctx.Done(): + t.Fatal("job did not start") + } + if err := running.Cancel(); err != nil { + t.Fatal(err) + } + if err := running.WaitContext(ctx); err != nil { + t.Fatal(err) + } + if _, err := running.Result(); !errors.Is(err, varmq.ErrJobCancelled) { + t.Errorf("running job: got %v, want ErrJobCancelled", err) + } + close(ran) + for v := range ran { + if v == 7 { + t.Error("The cancelled waiting job ran") + } + } +} + +// A pop drops at most maxDrops marked items; at the cap it reports items +// left, and the next pop goes on. +func TestCancelDropCap(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + for i := range maxDrops { + id := "m" + strconv.Itoa(i) + q.Enqueue(job(id)) + q.CancelJob(id) + } + q.Enqueue([]byte("plain")) + + items, acks := make([]any, 1), make([]string, 1) + if n, left, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != 0 || left != 1 { + t.Fatalf("at the cap: got n=%d left=%d err=%v, want 0, 1", n, left, err) + } + if v, _ := pop(t, q); v != "plain" { + t.Errorf("got %s, want plain", v) + } + if n := countJobs(t, q); n != maxDrops { + t.Errorf("Expected %d cancelled records, got %d", maxDrops, n) + } + + // Single-item pops can't report items left: past the cap they pop again + for _, p := range []struct { + name string + pop func() (any, bool) + }{ + {"DequeueWithAckId", func() (any, bool) { v, ok, _ := q.DequeueWithAckId(); return v, ok }}, + {"Dequeue", q.Dequeue}, + } { + for i := range maxDrops + 1 { + id := p.name + strconv.Itoa(i) + q.Enqueue(job(id)) + q.CancelJob(id) + } + q.Enqueue([]byte("plain")) + if v, ok := p.pop(); !ok || string(v.([]byte)) != "plain" { + t.Errorf("%s past the cap: got %v %v, want plain", p.name, v, ok) + } + } +} + +// sweep trims records past their age and drops expired marks. +func TestSweep(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + old := time.Now().Add(-keepFailedAge).UnixMilli() + q.f.db.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('old', 'failed', ?)", old) + q.f.db.Exec("INSERT INTO "+q.jobs+" (job_id, state, finished_at) VALUES ('new', 'failed', ?)", time.Now().UnixMilli()) + q.f.db.Exec("INSERT INTO "+q.cancelled+" (job_id, expires_at) VALUES ('gone', ?)", old) + q.CancelJob("live") + q.sweep() + var marks int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks) + if countJobs(t, q) != 1 || marks != 1 { + t.Errorf("Expected 1 record and 1 mark left, got %d and %d", countJobs(t, q), marks) + } +} + +// Queues.Close stops polls and drops live watches and hooks without +// calling them. +func TestCloseDropsWatchesAndHooks(t *testing.T) { + q, qs := openQueue(t) + q2 := otherProcess(t, q) + ch := watch(q, "w") + ids := make(chan string, 1) + q.OnCancel(func(id string) { ids <- id }) + if err := qs.Close(); err != nil { + t.Fatal(err) + } + select { + case <-q.w.stop: + default: + t.Fatal("Expected Close to stop the poll") + } + q2.Enqueue(job("w")) + _, ack := pop(t, q2) + q2.Acknowledge(ack) + q2.CancelJob("c") + noOutcome(t, ch) + select { + case id := <-ids: + t.Errorf("hook called after Close with %q", id) + default: + } + if stop := q.OnCancel(func(string) { t.Error("hook on a closed queue") }); stop == nil { + t.Error("Expected a stop func") + } +} + +// A hook added while a poll reads gets the marks made after its call; a +// stopped hook gets none; a mark whose job finished (stale) reaches nobody. +func TestOnCancelHooksDuringPoll(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + a, b := make(chan string, 4), make(chan string, 4) + stopA := q.OnCancel(func(id string) { a <- id }) + q.w.mu.Lock() + seq := q.w.lastSeq // a poll's snapshot, taken before B + q.w.mu.Unlock() + q.CancelJob("before-b") + stopB := q.OnCancel(func(id string) { b <- id }) + q.CancelJob("x") + q.pollCancels(seq) + + // The queue's own poll may report a mark too: compare distinct ids + got := func(ch chan string) []string { + var ids []string + for { + select { + case id := <-ch: + ids = append(ids, id) + case <-time.After(4 * pollEvery): + slices.Sort(ids) + return slices.Compact(ids) + } + } + } + if ids := got(a); !slices.Equal(ids, []string{"before-b", "x"}) { + t.Errorf("hook A: got %v", ids) + } + if ids := got(b); !slices.Equal(ids, []string{"x"}) { + t.Errorf("hook B: got %v, want only the mark after its call", ids) + } + + var xSeq int64 + q.f.db.QueryRow("SELECT seq FROM " + q.cancelled + " WHERE job_id = 'x'").Scan(&xSeq) + stopA() + q.Enqueue(job("y")) + _, ack := pop(t, q) + q.CancelJob("y") + q.Acknowledge(ack) // the finish drops the mark: stale + q.CancelJob("z") + q.pollCancels(xSeq) + if ids := got(a); len(ids) != 0 { + t.Errorf("stopped hook got %v", ids) + } + if ids := got(b); !slices.Equal(ids, []string{"z"}) { + t.Errorf("hook B: got %v, want z only (y is stale)", ids) + } + stopB() +} + +// A repeat CancelJob gets a new seq and expiry, so hooks hear it again. +func TestCancelRepeatRenotifies(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + ids := make(chan string, 4) + defer q.OnCancel(func(id string) { ids <- id })() + read := func() (seq, exp int64) { + q.f.db.QueryRow("SELECT seq, expires_at FROM "+q.cancelled+" WHERE job_id = 'x'").Scan(&seq, &exp) + return + } + for i := range 2 { + q.CancelJob("x") + select { + case <-ids: + case <-time.After(5 * time.Second): + t.Fatalf("cancel %d not reported", i+1) + } + if i == 0 { + s1, e1 := read() + time.Sleep(2 * time.Millisecond) + defer func() { + if s2, e2 := read(); s2 <= s1 || e2 <= e1 { + t.Errorf("Expected a new seq and expiry: %d,%d -> %d,%d", s1, e1, s2, e2) + } + }() + } + } +} + +// A gated Dequeue that drops the last item returns empty and keeps the +// drop; the mark is gone after it. +func TestDequeueGatedEmpty(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.Enqueue(job("x")) + q.CancelJob("x") + ch := watch(q, "x") + if v, ok := q.Dequeue(); ok { + t.Fatalf("got %v", v) + } + if got := waitOutcome(t, ch); got.State != contract.JobCancelled { + t.Errorf("got %+v", got) + } + var marks int + q.f.db.QueryRow("SELECT COUNT(*) FROM " + q.cancelled).Scan(&marks) + if q.Len() != 0 || marks != 0 { + t.Errorf("Expected no item and no mark, got %d and %d", q.Len(), marks) + } +} + +// With WithRemoveOnComplete(false) each end keeps its row with its status, +// and the record keeps the result. +func TestKeptRowEnds(t *testing.T) { + q, qs := openQueue(t, WithRemoveOnComplete(false)) + defer qs.Close() + q.Enqueue(job("r")) + q.Enqueue(job("c")) + _, ackR := pop(t, q) + _, ackC := pop(t, q) + if !q.AcknowledgeResult(ackR, []byte(`"ok"`)) || !q.AcknowledgeCancelled(ackC, "stop") { + t.Fatal("finish failed") + } + if countRows(t, q.f.db, "status = 'completed'") != 1 || countRows(t, q.f.db, "status = 'cancelled'") != 1 { + t.Error("Expected one completed and one cancelled row") + } + if got := waitOutcome(t, watch(q, "r")); string(got.Result) != `"ok"` { + t.Errorf("got %+v", got) + } + if got := waitOutcome(t, watch(q, "c")); got.State != contract.JobCancelled || got.Error != "stop" { + t.Errorf("got %+v", got) + } +} + +// A read reaches only watches started before it: an earlier run's record, +// read before a re-add, never ends a watch made after. +func TestDeliverSkipsNewerWatches(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + ch := watch(q, "x") + q.w.mu.Lock() + gen := q.w.gen + q.w.mu.Unlock() + q.deliver(contract.JobOutcome{ID: "x", State: contract.JobCompleted}, gen-1) + noOutcome(t, ch) + q.deliver(contract.JobOutcome{ID: "x", State: contract.JobCompleted}, gen) + waitOutcome(t, ch) +} + +// Every watch fires when far more jobs end between two polls than +// retention keeps, and more ids are watched than one read used to take. +func TestWatchBurst(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q2 := otherProcess(t, q) + const n = 3 * keepCompleted + var fired atomic.Int64 + done := make(chan struct{}) + for i := range n { + id := "b" + strconv.Itoa(i) + q.WatchOutcome(id, func(contract.JobOutcome) { + if fired.Add(1) == n { + close(done) + } + }) + q.Enqueue(job(id)) + } + items, acks := make([]any, 500), make([]string, 500) + for { + k, _, err := q2.DequeueBatchWithAckIds(items, acks) + if err != nil || k == 0 { + break + } + for _, a := range acks[:k] { + q2.Acknowledge(a) + } + } + select { + case <-done: + case <-time.After(20 * time.Second): + t.Fatalf("%d of %d watches fired", fired.Load(), n) + } +} + +// Queue names may not hold __sqliteq_, which names a queue's own tables. +func TestReservedQueueName(t *testing.T) { + qs := mustNew(t, filepath.Join(t.TempDir(), "one.db"), WithSingleFile()) + defer qs.Close() + if _, err := qs.NewQueue("orders__sqliteq_jobs"); err == nil { + t.Error("Expected a reserved name to fail") + } +} + +// An item whose envelope does not decode to a string id is a plain job: +// with a live mark (gated pops) it is handed out, finishes as usual, and +// writes no record. +func TestUndecodableEnvelopeIsPlainJob(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + q.CancelJob("other") // keeps the pops gated + bad := []string{`{bad`, `{"id":5}`, `{"id":null}`, `{"id":""}`, `[1]`, "plain", `{"id":"x"`} + for _, b := range bad { + q.Enqueue([]byte(b)) + } + for i, b := range bad { + v, ack := pop(t, q) + if v != b { + t.Fatalf("pop %d: got %q, want %q", i, v, b) + } + if i%2 == 0 { + q.Acknowledge(ack) + } else { + q.Reject(ack, "r") + } + } + if n := countJobs(t, q); n != 0 { + t.Errorf("Expected no records, got %d", n) + } +} diff --git a/registry.go b/registry.go index 24aeb0e..5db083a 100644 --- a/registry.go +++ b/registry.go @@ -62,8 +62,11 @@ func driverName(pages int) string { // dsn returns the connection string for a database file: WAL, relaxed // fsync (safe under WAL) and a busy timeout so writers wait for each other. +// Transactions begin IMMEDIATE: one that reads before it writes (CancelJob, +// a gated pop) would otherwise fail with SQLITE_BUSY, not wait, when another +// process wrote in between. func dsn(path string) string { - return path + "?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000" + return path + "?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=5000&_txlock=immediate" } // registry holds every open file in the process, keyed by resolved path. @@ -121,6 +124,10 @@ func (f *file) release() error { if registry.files[f.key] == f { delete(registry.files, f.key) } + // Stop the queues' polls before the pool closes under them + for _, h := range f.handles { + h.queue().Close() + } registry.mu.Unlock() // Close waits for running queries, so it runs outside the lock @@ -143,6 +150,10 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) } key := foldName(name) + // A queue's indexes, sidecar tables and trigger are named __sqliteq_* + if strings.Contains(key, "__sqliteq_") { + return nil, fmt.Errorf("sqliteq: queue name %q must not contain __sqliteq_", name) + } if h, ok := f.handles[key]; ok { if (h.pq != nil) != priority { return nil, fmt.Errorf("sqliteq: %q is already open as a %s", name, h.kind())