From c3431f0fbf47dbe283e919742750202d685b265e Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 20:23:51 +0600 Subject: [PATCH 01/10] test: add claim, ack and batch pop benchmarks --- bench_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 bench_test.go diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 0000000..ee3a418 --- /dev/null +++ b/bench_test.go @@ -0,0 +1,67 @@ +package sqliteq + +import ( + "fmt" + "testing" +) + +// benchQueue opens a queue with n pending rows, filled in one statement. +func benchQueue(b *testing.B, n int) *Queue { + b.Helper() + qs, err := New(b.TempDir()) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { qs.Close() }) + q, err := qs.NewQueue("bench") + if err != nil { + b.Fatal(err) + } + if _, err := q.f.db.Exec(fmt.Sprintf(`INSERT INTO %s (data, status, ack, created_at, updated_at) + WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < ?) + SELECT x'6a6f62', 'pending', 0, 0, 0 FROM c`, q.table), n); err != nil { + b.Fatal(err) + } + return q +} + +// backlog is the pending rows left behind the ones a benchmark takes +const backlog = 100_000 + +func BenchmarkClaim(b *testing.B) { + q := benchQueue(b, b.N+backlog) + b.ResetTimer() + for range b.N { + if _, ok, _ := q.DequeueWithAckId(); !ok { + b.Fatal("empty") + } + } +} + +func BenchmarkAck(b *testing.B) { + q := benchQueue(b, b.N+backlog) + items, acks := make([]any, b.N), make([]string, b.N) + if n, _, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != b.N { + b.Fatal(n, err) + } + b.ResetTimer() + for _, id := range acks { + if !q.Acknowledge(id) { + b.Fatal("not acknowledged") + } + } +} + +// BenchmarkBatchPop pops in batches of 64; one op is one item. +func BenchmarkBatchPop(b *testing.B) { + q := benchQueue(b, b.N+backlog) + items, acks := make([]any, 64), make([]string, 64) + b.ResetTimer() + for left := b.N; left > 0; { + n, _, err := q.DequeueBatchWithAckIds(items[:min(64, left)], acks) + if err != nil || n == 0 { + b.Fatal(n, err) + } + left -= n + } +} From 766053e3e1982dfa5b507b68842751a555486908 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 20:23:51 +0600 Subject: [PATCH 02/10] feat: recover by owner and heartbeat, not on open Each file in a process gets an owner id, stored on the rows it claims. A claiming process keeps a heartbeat row; every process with a queue open sweeps and requeues only rows of owners without a live heartbeat. A finish matches the owner. The last release hands own rows back. Refs #27 --- migration_test.go | 7 +- option.go | 25 ++- owner.go | 253 +++++++++++++++++++++ owner_test.go | 543 +++++++++++++++++++++++++++++++++++++++++++++ pause.go | 27 ++- queue.go | 37 ++- queue_test.go | 14 +- queues.go | 6 +- registry.go | 63 +++++- workflow.go | 2 +- workflow_finish.go | 4 +- 11 files changed, 932 insertions(+), 49 deletions(-) create mode 100644 owner.go create mode 100644 owner_test.go diff --git a/migration_test.go b/migration_test.go index 8e70673..8332fd7 100644 --- a/migration_test.go +++ b/migration_test.go @@ -129,11 +129,14 @@ func TestMigrateV02File(t *testing.T) { dequeue, length, db = q.Dequeue, q.Len, q.f.db } - want := []string{name + jobIDIndexSuffix, name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} + want := []string{name + jobIDIndexSuffix, name + pendingIndexSuffix, name + processingIndexSuffix, "sqlite_autoindex_" + name + "_1"} if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { t.Errorf("Round %d: indexes = %v, want %v", round, got, want) } - for _, side := range []string{name + jobsSuffix, name + cancelledSuffix, name + readdSuffix} { + if ok, err := hasColumn(db, name, "owner"); err != nil || !ok { + t.Errorf("Round %d: expected the owner column on the old table, got %t %v", round, ok, err) + } + for _, side := range []string{name + jobsSuffix, name + cancelledSuffix, name + readdSuffix, "__sqliteq_owners", "__sqliteq_clock"} { var n int if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = ?", side).Scan(&n); err != nil || n != 1 { t.Errorf("Round %d: expected %s on the old file, got %d %v", round, side, n, err) diff --git a/option.go b/option.go index c56379f..2b39b4c 100644 --- a/option.go +++ b/option.go @@ -1,6 +1,9 @@ package sqliteq -import "errors" +import ( + "errors" + "time" +) // errNilOption is returned by New, NewQueue and NewPriorityQueue for a nil // option, which they would otherwise call @@ -21,10 +24,11 @@ func WithRemoveOnComplete(remove bool) Option { type storeConfig struct { singleFile bool checkpointPages int + heartbeat time.Duration } func defaultStoreConfig() storeConfig { - return storeConfig{checkpointPages: 1000} + return storeConfig{checkpointPages: 1000, heartbeat: defaultHeartbeat} } // StoreOption is a function type that can be used to configure a store passed to New @@ -44,3 +48,20 @@ func WithCheckpointPages(n int) StoreOption { c.checkpointPages = n } } + +// WithHeartbeatTimeout sets how long this process may go without renewing +// its heartbeat in a database file before other processes on the file take +// it for dead and requeue the items it dequeued and has not acknowledged +// (default 30s, at least 5s; New rejects anything lower). The heartbeat is +// renewed every sixth of the timeout, so it runs out only when the process +// dies or freezes. It does not limit how long a job may run. +// +// A renewal waits up to 5s (the busy timeout) behind another process's +// write, so keep the timeout well above that on a file other processes +// write to. A job still running when its process's heartbeat runs out runs +// again elsewhere. Processes with different timeouts can share a file. +func WithHeartbeatTimeout(d time.Duration) StoreOption { + return func(c *storeConfig) { + c.heartbeat = d + } +} diff --git a/owner.go b/owner.go new file mode 100644 index 0000000..4995d32 --- /dev/null +++ b/owner.go @@ -0,0 +1,253 @@ +package sqliteq + +import ( + "crypto/rand" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "time" +) + +// Owners and heartbeats (redisq's model, on SQLite). Each file in a process +// has a random owner id, stored on every row it claims. An owner that +// claims keeps a row in __sqliteq_owners, renewed by the sweep; a process +// that never claims writes none. The sweep runs on a table's first open and +// then every hb/6 for every table the process has open: it requeues the +// processing rows whose owner has no live row. SQLite has no presence +// signal, so the heartbeat is the only liveness: a killed owner's rows come +// back one timeout after its last beat. +// +// Times are the host's wall clock in Unix ms. __sqliteq_clock holds when +// any process last swept; see sweep for the gap guard. + +const ( + defaultHeartbeat = 30 * time.Second + // minHeartbeat is the shortest timeout New takes. A beat can wait the + // 5s busy timeout behind another process's write, so on a file with + // writers in other processes the timeout must stay well above 5s. + minHeartbeat = 5 * time.Second + + processingIndexSuffix = "__sqliteq_processing" + + // The time is read by SQLite once the statement holds the write lock, + // so a wait for another process does not shorten the heartbeat + beatQuery = `INSERT INTO __sqliteq_owners (id, expires_at) VALUES (?, CAST(unixepoch('subsec') * 1000 AS INTEGER) + ?) + ON CONFLICT (id) DO UPDATE SET expires_at = excluded.expires_at` +) + +func newOwnerID() string { + var b [8]byte + rand.Read(b[:]) // never fails + return hex.EncodeToString(b[:]) +} + +// sweepEvery is the sweep interval; gapLimit the longest gap between two +// sweeps of the file that counts as normal. +// ponytail: the limit comes from this process's own timeout. A process with +// a short timeout that opens a file swept at a longer interval moves the +// heartbeats on once, which delays a reap by that gap. Upgrade path: keep +// the sweepers' interval in the clock row. +func (f *file) sweepEvery() time.Duration { return f.hb / 6 } +func (f *file) gapLimit() int64 { return (f.hb / 3).Milliseconds() } + +// initOwners creates the file's owner and clock tables. The caller holds +// the write mutex. +func (f *file) initOwners() error { + for _, s := range []string{ + `CREATE TABLE IF NOT EXISTS __sqliteq_owners (id TEXT PRIMARY KEY, expires_at INTEGER NOT NULL)`, + `CREATE TABLE IF NOT EXISTS __sqliteq_clock (id INTEGER PRIMARY KEY CHECK (id = 1), swept_at INTEGER NOT NULL)`, + } { + if _, err := f.db.Exec(s); err != nil { + return err + } + } + return nil +} + +// addOwner adds the owner column a claim fills, when it is missing, the +// index the sweep reads, and the file's owner tables. Pending rows and rows +// of older versions are NULL. +func (q *Queue) addOwner() error { + exists, err := hasColumn(q.f.db, q.name, "owner") + if err != nil { + return err + } + if !exists { + if _, err := q.f.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN owner TEXT", q.table)); err != nil { + return err + } + } + // Without it a sweep scans the whole table, pending backlog included. + // Keyed by id: the smallest entry for a claim and an ack to maintain. + if _, err := q.f.db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (id) WHERE status = 'processing'", + quoteIdent(q.name+processingIndexSuffix), q.table)); err != nil { + return err + } + return q.f.initOwners() +} + +// beat writes this process's owner row before a claim when it has none, or +// when its last write is older than two sweep intervals: a process that +// froze must not hand out rows under an owner that already looks dead. The +// caller holds the write mutex. +func (f *file) beat() error { + if f.closing { + return errClosed + } + if f.claimed && time.Now().UnixMilli()-f.lastBeat <= 2*f.sweepEvery().Milliseconds() { + return nil + } + if _, err := f.db.Exec(beatQuery, f.owner, f.hb.Milliseconds()); err != nil { + return err + } + f.claimed, f.lastBeat = true, time.Now().UnixMilli() + return nil +} + +// sweep is one write transaction: it moves the clock, renews this process's +// owner row if it has one, and puts back to pending the unacknowledged +// processing rows of qs whose owner has no live row; with legacy also those +// without an owner, which an older version claimed. It returns the queues +// whose rows moved. +// +// Gap guard: a gap since the last sweep by any process longer than gapLimit +// means no process could beat (a suspended host, a stopped process), and a +// gap below -gapLimit that the clock was set back. Every heartbeat moves on +// by the gap first, so a stall takes no owner for dead at once. With no +// clock row yet, no owner is taken for dead. +func (f *file) sweep(qs []*Queue, legacy bool) (moved []*Queue, err error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.closing { + return nil, errClosed + } + + tx, err := f.db.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + // Read after Begin, which can wait for another process's write + now := time.Now().UnixMilli() + var swept int64 + reap := true + switch err := tx.QueryRow("SELECT swept_at FROM __sqliteq_clock").Scan(&swept); { + case errors.Is(err, sql.ErrNoRows): + reap = false + case err != nil: + return nil, err + default: + if gap := now - swept; gap > f.gapLimit() || gap < -f.gapLimit() { + if _, err := tx.Exec("UPDATE __sqliteq_owners SET expires_at = expires_at + ?", gap); err != nil { + return nil, err + } + } + } + if _, err := tx.Exec(`INSERT INTO __sqliteq_clock (id, swept_at) VALUES (1, ?) + ON CONFLICT (id) DO UPDATE SET swept_at = excluded.swept_at`, now); err != nil { + return nil, err + } + if f.claimed { + if _, err := tx.Exec(beatQuery, f.owner, f.hb.Milliseconds()); err != nil { + return nil, err + } + } + + // NULL NOT IN an empty set is true, so the ticker sweep names NOT NULL + const dead = "owner NOT IN (SELECT id FROM __sqliteq_owners WHERE expires_at > ?)" + var where string + var args []any + switch { + case reap && legacy: + where, args = "(owner IS NULL OR "+dead+")", []any{now} + case reap: + where, args = "owner IS NOT NULL AND "+dead, []any{now} + case legacy: + where = "owner IS NULL" + } + if where != "" { + at := time.UnixMilli(now).UTC() + for _, q := range qs { + res, err := tx.Exec(fmt.Sprintf("UPDATE %s SET status = 'pending', owner = NULL, updated_at = ? WHERE status = 'processing' AND ack = 0 AND %s", + q.table, where), append([]any{at}, args...)...) + if err != nil { + return nil, err + } + if n, _ := res.RowsAffected(); n > 0 { + moved = append(moved, q) + } + } + } + if reap { + if _, err := tx.Exec("DELETE FROM __sqliteq_owners WHERE expires_at <= ?", now); err != nil { + return nil, err + } + } + if err := tx.Commit(); err != nil { + return nil, err + } + if f.claimed { + f.lastBeat = now + } + return moved, nil +} + +// sweepLoop sweeps every table the process has open on the file, every +// sweepEvery until the last release, and wakes the queues whose rows moved. +// It runs in every process with a queue open, also one that never claims: +// VarMQ pops only when Len is above 0, so a consumer that waits on an empty +// queue would never start it, and a dead owner's rows would stay. +func (f *file) sweepLoop() { + defer close(f.done) + t := time.NewTicker(f.sweepEvery()) + defer t.Stop() + for { + select { + case <-f.stop: + return + case <-t.C: + } + + registry.mu.Lock() + qs := make([]*Queue, 0, len(f.handles)) + for _, h := range f.handles { + qs = append(qs, h.queue()) + } + registry.mu.Unlock() + + // A failed sweep (a busy file) is tried again at the next tick + moved, _ := f.sweep(qs, false) + for _, q := range moved { + q.wake() + } + } +} + +// handBack puts this owner's unacknowledged rows in tables back to pending +// and drops its owner row, on the last release: a clean exit returns its +// jobs at once. If it fails, they return one timeout later. +func (f *file) handBack(tables []string) { + f.mu.Lock() + defer f.mu.Unlock() + if !f.claimed { + return + } + tx, err := f.db.Begin() + if err != nil { + return + } + defer tx.Rollback() + now := time.Now().UTC() + for _, t := range tables { + if _, err := tx.Exec(fmt.Sprintf("UPDATE %s SET status = 'pending', owner = NULL, updated_at = ? WHERE status = 'processing' AND ack = 0 AND owner = ?", t), + now, f.owner); err != nil { + return + } + } + if _, err := tx.Exec("DELETE FROM __sqliteq_owners WHERE id = ?", f.owner); err != nil { + return + } + tx.Commit() +} diff --git a/owner_test.go b/owner_test.go new file mode 100644 index 0000000..52acf75 --- /dev/null +++ b/owner_test.go @@ -0,0 +1,543 @@ +package sqliteq + +import ( + "bufio" + "database/sql" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/goptics/varmq" +) + +// requeueNoAck is what a restart finds: this process's owner is dead, so a +// sweep puts the rows it claimed back to pending. +func (q *Queue) requeueNoAck() error { + f := q.f + f.mu.Lock() + f.claimed = false + _, err := f.db.Exec("DELETE FROM __sqliteq_owners WHERE id = ?", f.owner) + f.mu.Unlock() + if err != nil { + return err + } + _, err = f.sweep([]*Queue{q}, true) + return err +} + +// fastStore is a store whose files get heartbeat timeout hb, below what New +// takes. +func fastStore(t *testing.T, path string, hb time.Duration, opts ...StoreOption) Queues { + t.Helper() + qs := mustNew(t, path, opts...) + qs.(*queues).cfg.heartbeat = hb + t.Cleanup(func() { qs.Close() }) + return qs +} + +// variants are the queue kinds and layouts every owner case runs on. +var variants = []struct { + name string + priority, single bool + opts []Option +}{ + {name: "fifo"}, + {name: "priority", priority: true}, + {name: "keep", opts: []Option{WithRemoveOnComplete(false)}}, + {name: "single file", single: true}, +} + +// eachVariant runs fn on queue "q" of each variant, opened with timeout hb. +func eachVariant(t *testing.T, hb time.Duration, fn func(t *testing.T, q *Queue, qs Queues)) { + for _, v := range variants { + t.Run(v.name, func(t *testing.T) { + path := t.TempDir() + var so []StoreOption + if v.single { + path, so = filepath.Join(path, "one.db"), []StoreOption{WithSingleFile()} + } + qs := fastStore(t, path, hb, so...) + if v.priority { + pq, err := qs.NewPriorityQueue("q", v.opts...) + if err != nil { + t.Fatal(err) + } + fn(t, pq.Queue, qs) + return + } + q, err := qs.NewQueue("q", v.opts...) + if err != nil { + t.Fatal(err) + } + fn(t, q, qs) + }) + } +} + +// kill stops f as a killed process stops: no hand-back, its owner row and +// its processing rows stay. A later release closes nothing twice. +func kill(f *file) { + f.mu.Lock() + f.closing = true + f.mu.Unlock() + registry.mu.Lock() + sweeping := f.sweeping + f.sweeping = false + registry.mu.Unlock() + if sweeping { + close(f.stop) + <-f.done + } + f.db.Close() +} + +// claim1 adds one item to q and claims it. +func claim1(t *testing.T, q *Queue) (ack string) { + t.Helper() + if !q.enqueue([]byte("job"), 0, "") { + t.Fatal("enqueue failed") + } + _, ok, ack := q.DequeueWithAckId() + if !ok { + t.Fatal("claim failed") + } + return ack +} + +// rowOf reads the status and owner of table q's one row through db. +func rowOf(t *testing.T, db *sql.DB) (status string, owner sql.NullString) { + t.Helper() + if err := db.QueryRow(`SELECT status, owner FROM "q"`).Scan(&status, &owner); err != nil { + t.Fatal(err) + } + return status, owner +} + +func ownerRows(t *testing.T, db *sql.DB) (n int) { + t.Helper() + if err := db.QueryRow("SELECT COUNT(*) FROM __sqliteq_owners").Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func eventually(t *testing.T, what string, cond func() bool) { + t.Helper() + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { + if cond() { + return + } + } + t.Fatalf("timed out: %s", what) +} + +func mustExec(t *testing.T, db *sql.DB, query string, args ...any) { + t.Helper() + if _, err := db.Exec(query, args...); err != nil { + t.Fatal(err) + } +} + +// A second process that opens the table and sweeps leaves a live owner's +// row alone. +func TestOwnerLive(t *testing.T) { + const hb = 300 * time.Millisecond + eachVariant(t, hb, func(t *testing.T, a *Queue, qs Queues) { + ack := claim1(t, a) + b := otherProcess(t, a) + time.Sleep(2 * hb) // both sweep, a beats + if status, owner := rowOf(t, b.f.db); status != "processing" || owner.String != a.f.owner { + t.Fatalf("Expected a's row processing under %s, got %s under %q", a.f.owner, status, owner.String) + } + if n := b.Len(); n != 0 { + t.Fatalf("Expected nothing pending for b, got %d", n) + } + if !a.Acknowledge(ack) { + t.Fatal("Expected a's ack to succeed") + } + }) +} + +// A survivor that opened first and never claims gets a killed owner's row +// back within one timeout, with no reopen, and its hooks are woken. +func TestOwnerDead(t *testing.T) { + const hb = 300 * time.Millisecond + eachVariant(t, hb, func(t *testing.T, a *Queue, qs Queues) { + woke := make(chan bool, 16) + defer a.OnPauseChange(func(p bool) { woke <- p })() + b := otherProcess(t, a) + claim1(t, b) + kill(b.f) + if status, _ := rowOf(t, a.f.db); status != "processing" { + t.Fatalf("Expected the row processing right after the kill, got %s", status) + } + start := time.Now() + eventually(t, "dead owner's row pending", func() bool { return a.Len() == 1 }) + if d := time.Since(start); d > 5*time.Second { + t.Errorf("Recovery took %s", d) + } + if _, owner := rowOf(t, a.f.db); owner.Valid { + t.Errorf("Expected the owner cleared, got %q", owner.String) + } + select { + case p := <-woke: + if p { + t.Error("Expected the wake to report running") + } + case <-time.After(5 * time.Second): + t.Error("Expected the hooks woken after the requeue") + } + if a.f.claimed { + t.Error("Expected the survivor to have no owner row before its first claim") + } + _, ok, ack := a.DequeueWithAckId() + if !ok || !a.Acknowledge(ack) { + t.Fatal("Expected the survivor to run the recovered job") + } + eventually(t, "dead owner's row dropped", func() bool { return ownerRows(t, a.f.db) == 1 }) + }) +} + +// After a crash the restarted process gets its old rows back one timeout +// later, not at open. +func TestOwnerRestart(t *testing.T) { + const hb = 300 * time.Millisecond + dir := t.TempDir() + qs := fastStore(t, dir, hb) + a, err := qs.NewQueue("q") + if err != nil { + t.Fatal(err) + } + claim1(t, a) + kill(a.f) + qs.Close() + + qs = fastStore(t, dir, hb) + a, err = qs.NewQueue("q") + if err != nil { + t.Fatal(err) + } + if n := a.Len(); n != 0 { + t.Fatalf("Expected the row to wait for the old heartbeat, got %d pending", n) + } + eventually(t, "old row pending", func() bool { return a.Len() == 1 }) +} + +// An ack or reject from the owner that lost a row does not end the run of +// the owner that holds it now. +func TestStaleAck(t *testing.T) { + eachVariant(t, defaultHeartbeat, func(t *testing.T, a *Queue, qs Queues) { + ack := claim1(t, a) + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = 0 WHERE id = ?", a.f.owner) // a froze + b := otherProcess(t, a) // its first sweep requeues + _, ok, ackB := b.DequeueWithAckId() + if !ok || ackB != ack { + t.Fatalf("Expected b to claim the same row, got %t %q", ok, ackB) + } + if a.Acknowledge(ack) || a.Reject(ack, "boom") { + t.Fatal("Expected a's stale ack and reject to fail") + } + if found, err := a.finishDirect(ack, "completed", "", nil); found || err != nil { + t.Fatalf("Expected a's stale direct finish to find nothing, got %t %v", found, err) + } + if status, owner := rowOf(t, a.f.db); status != "processing" || owner.String != b.f.owner { + t.Fatalf("Expected the row processing under b, got %s under %q", status, owner.String) + } + if !b.Acknowledge(ackB) { + t.Fatal("Expected b's ack to succeed") + } + }) +} + +// A processing row without an owner, from an older version, goes back on a +// table's first open only. +func TestLegacyRows(t *testing.T) { + a, qs := openQueue(t) + defer qs.Close() + mustExec(t, a.f.db, `INSERT INTO "q" (data, status, ack) VALUES (x'00', 'processing', 0)`) + if _, err := a.f.sweep([]*Queue{a}, false); err != nil { + t.Fatal(err) + } + if status, _ := rowOf(t, a.f.db); status != "processing" { + t.Fatalf("Expected the ticker sweep to leave the row, got %s", status) + } + otherProcess(t, a) + if status, _ := rowOf(t, a.f.db); status != "pending" { + t.Fatalf("Expected a first open to requeue the row, got %s", status) + } +} + +// The last Close hands this owner's rows back, drops its owner row and +// stops the sweep. +func TestCleanClose(t *testing.T) { + eachVariant(t, defaultHeartbeat, func(t *testing.T, a *Queue, qs Queues) { + ack := claim1(t, a) + claim1(t, a) + if !a.Acknowledge(ack) { + t.Fatal("ack failed") + } + f := a.f + kept := !a.removeOnComplete + if err := qs.Close(); err != nil { + t.Fatal(err) + } + select { + case <-f.done: + default: + t.Fatal("Expected the sweep loop stopped") + } + + db, err := sql.Open("sqlite3", f.key) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var pending, owned int + if err := db.QueryRow(`SELECT COUNT(*) FILTER (WHERE status = 'pending'), COUNT(owner) FILTER (WHERE status <> 'completed') FROM "q"`).Scan(&pending, &owned); err != nil { + t.Fatal(err) + } + if pending != 1 || owned != 0 || ownerRows(t, db) != 0 { + t.Fatalf("Expected 1 pending row, no owner (kept %t), got %d pending, %d owned, %d owner rows", kept, pending, owned, ownerRows(t, db)) + } + }) +} + +// A long gap since the last sweep, a clock set back, or no clock row takes +// no owner for dead. +func TestClockGap(t *testing.T) { + setup := func(t *testing.T) (a *Queue, dead string) { + a, qs := openQueue(t) + t.Cleanup(func() { qs.Close() }) + b := otherProcess(t, a) + claim1(t, b) + kill(b.f) + return a, b.f.owner + } + sweep := func(t *testing.T, a *Queue) (status string, expires int64) { + t.Helper() + if _, err := a.f.sweep([]*Queue{a}, false); err != nil { + t.Fatal(err) + } + status, _ = rowOf(t, a.f.db) + a.f.db.QueryRow("SELECT expires_at FROM __sqliteq_owners").Scan(&expires) + return status, expires + } + const hour = int64(time.Hour / time.Millisecond) + + t.Run("long gap", func(t *testing.T) { + a, dead := setup(t) + now := time.Now().UnixMilli() + mustExec(t, a.f.db, "UPDATE __sqliteq_clock SET swept_at = ?", now-hour) + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = ? WHERE id = ?", now-hour+1000, dead) + if status, expires := sweep(t, a); status != "processing" || expires < now { + t.Fatalf("Expected the owner kept after a long gap, got %s, expires %d ms from now", status, expires-now) + } + // Its second left runs out as usual + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = expires_at - 2000") + if status, _ := sweep(t, a); status != "pending" || ownerRows(t, a.f.db) != 0 { + t.Fatalf("Expected the dead owner reaped, got %s, %d owner rows", status, ownerRows(t, a.f.db)) + } + }) + t.Run("clock set back", func(t *testing.T) { + a, dead := setup(t) + now := time.Now().UnixMilli() + mustExec(t, a.f.db, "UPDATE __sqliteq_clock SET swept_at = ?", now+hour) + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = ? WHERE id = ?", now+hour+1000, dead) + status, expires := sweep(t, a) + if status != "processing" || expires < now || expires > now+10_000 { + t.Fatalf("Expected the heartbeat moved back with the clock, got %s, expires %d ms from now", status, expires-now) + } + }) + t.Run("no clock row", func(t *testing.T) { + a, _ := setup(t) + mustExec(t, a.f.db, "DELETE FROM __sqliteq_clock") + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = 0") + if status, _ := sweep(t, a); status != "processing" { + t.Fatalf("Expected no owner taken for dead without a clock, got %s", status) + } + if status, _ := sweep(t, a); status != "pending" { + t.Fatalf("Expected the next sweep to reap, got %s", status) + } + }) +} + +// A process that never claims writes no owner row. +func TestProducerOnly(t *testing.T) { + a, qs := openQueue(t) + defer qs.Close() + a.Enqueue([]byte("x")) + a.Len() + a.Values() + if a.Pause() != nil || a.Resume() != nil { + t.Fatal("pause failed") + } + stats(t, a) + a.Dequeue() + if _, err := a.f.sweep([]*Queue{a}, false); err != nil { + t.Fatal(err) + } + if n := ownerRows(t, a.f.db); n != 0 || a.f.claimed { + t.Fatalf("Expected no owner row for a producer, got %d", n) + } + a.DequeueWithAckId() // an empty claim is a consumer's + if n := ownerRows(t, a.f.db); n != 1 { + t.Fatalf("Expected the first claim to write the owner row, got %d", n) + } +} + +// A process that froze past two intervals beats before it claims again. +func TestBeatAfterStall(t *testing.T) { + a, qs := openQueue(t) + defer qs.Close() + claim1(t, a) + a.f.mu.Lock() + a.f.lastBeat -= time.Hour.Milliseconds() + a.f.mu.Unlock() + mustExec(t, a.f.db, "DELETE FROM __sqliteq_owners") // another process reaped it + claim1(t, a) + var expires int64 + if err := a.f.db.QueryRow("SELECT expires_at FROM __sqliteq_owners WHERE id = ?", a.f.owner).Scan(&expires); err != nil { + t.Fatalf("Expected the owner row back before the claim: %v", err) + } + if left := expires - time.Now().UnixMilli(); left < 20_000 || left > 31_000 { + t.Errorf("Expected about one timeout left, got %d ms", left) + } +} + +func TestHeartbeatOption(t *testing.T) { + dir := t.TempDir() + if _, err := New(dir, WithHeartbeatTimeout(4*time.Second)); err == nil { + t.Error("Expected New to refuse a timeout under 5s") + } + qs := mustNew(t, dir) + defer qs.Close() + q, err := qs.NewQueue("q") + if err != nil || q.f.hb != 30*time.Second { + t.Fatalf("Expected the 30s default, got %v %v", q.f.hb, err) + } + other := mustNew(t, dir, WithHeartbeatTimeout(10*time.Second)) + defer other.Close() + if _, err := other.NewQueue("q"); err == nil || !strings.Contains(err.Error(), "heartbeat timeout") { + t.Errorf("Expected a file open under another timeout refused, got %v", err) + } + if q2, err := other.NewQueue("fresh"); err != nil || q2.f.hb != 10*time.Second { + t.Errorf("Expected 10s on a new file, got %v", err) + } +} + +// The sweep and the hand-back read the processing index, not the pending +// backlog, and a finish stays a rowid lookup. +func TestOwnerQueryPlans(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + const set = `UPDATE "q" SET status = 'pending', owner = NULL WHERE status = 'processing' AND ack = 0 AND ` + for query, want := range map[string]string{ + set + "(owner IS NULL OR owner NOT IN (SELECT id FROM __sqliteq_owners WHERE expires_at > 0))": "USING INDEX q__sqliteq_processing", + set + "owner IS NOT NULL AND owner NOT IN (SELECT id FROM __sqliteq_owners WHERE expires_at > 0)": "USING INDEX q__sqliteq_processing", + set + "owner = 'x'": "USING INDEX q__sqliteq_processing", + `DELETE FROM "q" WHERE id = 1 AND status = 'processing' AND owner = 'x' RETURNING job_id, data`: "USING INTEGER PRIMARY KEY (rowid=?)", + } { + var id, parent, unused int + var detail string + if err := q.f.db.QueryRow("EXPLAIN QUERY PLAN "+query).Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatal(err) + } + if !strings.Contains(detail, want) { + t.Errorf("%s\n\tplan %q, want %q", query, detail, want) + } + } +} + +// TestHelperProcess is the second process of TestKilledProcess: a VarMQ +// worker that takes one job and never ends it. +func TestHelperProcess(t *testing.T) { + dir := os.Getenv("SQLITEQ_HELPER_DIR") + if dir == "" { + t.Skip("helper of TestKilledProcess") + } + qs := mustNew(t, dir) + qs.(*queues).cfg.heartbeat = time.Second + q, err := qs.NewQueue("q") + if err != nil { + t.Fatal(err) + } + w := varmq.NewWorker(func(j varmq.Job[int]) { + fmt.Println("claimed") + select {} + }) + w.WithQueue(q).Add(7) + select {} +} + +// Two real processes and a VarMQ worker in each: the job of a live process +// is not run twice, and the job of a killed one runs in the survivor, which +// opened first and never claimed, with no reopen. +func TestKilledProcess(t *testing.T) { + dir := t.TempDir() + qs := fastStore(t, dir, time.Second) + q, err := qs.NewQueue("q") + if err != nil { + t.Fatal(err) + } + + cmd := exec.Command(os.Args[0], "-test.run=^TestHelperProcess$") + cmd.Env = append(os.Environ(), "SQLITEQ_HELPER_DIR="+dir) + out, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + cmd.Process.Kill() + cmd.Wait() + }() + claimed := make(chan bool, 1) + go func() { + sc := bufio.NewScanner(out) + for sc.Scan() { + if sc.Text() == "claimed" { + claimed <- true + return + } + } + claimed <- false + }() + select { + case ok := <-claimed: + if !ok { + t.Fatal("helper ended before it claimed") + } + case <-time.After(30 * time.Second): + t.Fatal("helper did not claim") + } + + ran := make(chan int, 4) + w := varmq.NewWorker(func(j varmq.Job[int]) { ran <- j.Data() }) + defer w.Stop() + w.WithQueue(q) + select { + case v := <-ran: + t.Fatalf("job %d ran while its owner is alive", v) + case <-time.After(2500 * time.Millisecond): // well past one timeout + } + if status, _ := rowOf(t, q.f.db); status != "processing" { + t.Fatalf("Expected the live owner's row processing, got %s", status) + } + + cmd.Process.Kill() + cmd.Wait() + select { + case v := <-ran: + if v != 7 { + t.Errorf("Expected job 7, got %d", v) + } + case <-time.After(15 * time.Second): + t.Fatal("the killed owner's job did not run in the survivor") + } +} diff --git a/pause.go b/pause.go index 05388a2..917c46a 100644 --- a/pause.go +++ b/pause.go @@ -72,7 +72,8 @@ func (q *Queue) PauseRefusals() uint64 { // OnPauseChange implements varmq.PauseNotifier: fn gets the cache each time // this object sees it change (its own Pause or Resume, a refused pop, the -// poll), until cancel. fn runs on the goroutine that saw the change, never +// poll), until cancel. fn(false) also runs, with no change, when a sweep +// puts a dead owner's rows back in an unpaused queue (see wake). fn runs on the goroutine that saw the change, never // under a lock of this package, and must not block; racing changes can // reach fn out of order, never with a value the cache did not hold. func (q *Queue) OnPauseChange(fn func(paused bool)) (cancel func()) { @@ -124,6 +125,30 @@ func (q *Queue) setPaused(p bool, g uint64) { } } +// wake tells the OnPauseChange hooks of an unpaused queue that it runs: a +// sweep put rows back, and the hook is the one way to wake a VarMQ worker +// that waits on an empty queue. +// ponytail: borrows the pause hook, fn(false) with no change of the cache. +// Upgrade path: a wake interface of its own in VarMQ's contract. +func (q *Queue) wake() { + if q.closed.Load() { + return + } + q.pz.mu.Lock() + if q.pz.paused.Load() { + q.pz.mu.Unlock() + return + } + fns := make([]func(bool), 0, len(q.pz.hooks)) + for h := range q.pz.hooks { + fns = append(fns, *h) + } + q.pz.mu.Unlock() + for _, fn := range fns { + fn(false) + } +} + // refused records a pop refused for a pause, by a call that began at gen g. func (q *Queue) refused(g uint64) { q.pz.refusals.Add(1) diff --git a/queue.go b/queue.go index 6f1cbeb..638945e 100644 --- a/queue.go +++ b/queue.go @@ -115,6 +115,9 @@ func (q *Queue) initSchema() error { if err := q.addJobID(); err != nil { return fmt.Errorf("failed to add job_id: %w", err) } + if err := q.addOwner(); err != nil { + return fmt.Errorf("failed to add owner: %w", err) + } if err := q.initSidecars(); err != nil { return fmt.Errorf("failed to create outcome tables: %w", err) } @@ -241,10 +244,11 @@ func (q *Queue) prepare() (err error) { pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' AND NOT EXISTS (SELECT 1 FROM %s WHERE key = 'paused_at') ORDER BY %s", q.table, q.meta, q.order) - // fin ends a processing row: deleted, or kept with its end as status - fin := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table) + // fin ends a processing row this process claimed: deleted, or kept with + // its end as status. A row another owner took since is not touched. + fin := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing' AND owner = ? RETURNING job_id, data", q.table) if !q.removeOnComplete { - fin = fmt.Sprintf("UPDATE %s SET status = ?, ack = 1, updated_at = ? WHERE id = ? AND status = 'processing' RETURNING job_id, data", q.table) + fin = fmt.Sprintf("UPDATE %s SET status = ?, ack = 1, updated_at = ? WHERE id = ? AND status = 'processing' AND owner = ? RETURNING job_id, data", q.table) } for _, s := range []struct { @@ -254,7 +258,7 @@ func (q *Queue) prepare() (err error) { // The conflict target is the job_id index alone, so a NOT NULL or any // other failure still errors; it must repeat the index's WHERE {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING", q.table, cols, vals)}, - {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, + {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', owner = ?, updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data, job_id", q.table, pending)}, {&q.fin, fin}, {&q.putJob, fmt.Sprintf("INSERT OR REPLACE INTO %s (job_id, state, error, result, finished_at) VALUES (?, ?, ?, ?, ?)", q.jobs)}, @@ -284,18 +288,6 @@ func (q *Queue) closeStmts() { } } -// requeueNoAck puts rows left in processing without an ack back to pending -func (q *Queue) requeueNoAck() error { - q.f.mu.Lock() - defer q.f.mu.Unlock() - - _, err := q.f.db.Exec( - fmt.Sprintf("UPDATE %s SET status = 'pending', updated_at = ? WHERE status = 'processing' AND ack = 0", q.table), - time.Now().UTC(), - ) - return err -} - // hasColumn reports whether the table has a column with the given name func hasColumn(db *sql.DB, table, column string) (bool, error) { rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", quoteIdent(table))) @@ -486,7 +478,10 @@ func (q *Queue) pushWorkflow(item []byte, priority int, id string) error { func (q *Queue) claimN(n int) (out []claimed, capped, refused bool, err error) { g := q.pz.gen.Load() q.f.mu.Lock() - out, capped, refused, err = q.claimGated(n) + // The owner row is live before a row carries the owner + if err = q.f.beat(); err == nil { + out, capped, refused, err = q.claimGated(n) + } q.f.mu.Unlock() if refused { q.refused(g) @@ -575,7 +570,7 @@ func (q *Queue) claimGated(n int) (out []claimed, capped, refused bool, err erro // claimRows runs a claim statement and reads every row it returns; the // caller holds the write mutex func (q *Queue) claimRows(claim *sql.Stmt, n int) ([]claimed, error) { - rows, err := claim.Query(time.Now().UTC(), n) + rows, err := claim.Query(q.f.owner, time.Now().UTC(), n) if err != nil { return nil, err } @@ -800,13 +795,13 @@ func (q *Queue) finishTx(tx *sql.Tx, id int64, state contract.JobState, reason s // end finishes processing row id with state, in tx: the row is deleted or // kept with state as status, and a job with a Job ID gets its outcome // record. jobID is the row's, when the caller knows it. Reports whether -// the row was processing. +// the row was processing under this process's owner id. func (q *Queue) end(tx *sql.Tx, id int64, jobID string, state contract.JobState, reason string, result []byte, now time.Time) (bool, error) { var row *sql.Row if q.removeOnComplete { - row = tx.Stmt(q.fin).QueryRow(id) + row = tx.Stmt(q.fin).QueryRow(id, q.f.owner) } else { - row = tx.Stmt(q.fin).QueryRow(string(state), now.UTC(), id) + row = tx.Stmt(q.fin).QueryRow(string(state), now.UTC(), id, q.f.owner) } var ( once sql.NullString diff --git a/queue_test.go b/queue_test.go index 9881d5c..95439d7 100644 --- a/queue_test.go +++ b/queue_test.go @@ -914,22 +914,16 @@ func TestWritesTakeFileMutex(t *testing.T) { // process on the same database file would. func otherProcess(t *testing.T, q *Queue) *Queue { t.Helper() - db, err := sql.Open(driverName(q.f.pages), dsn(q.f.key)) + f, err := newFile(q.f.key, strings.TrimPrefix(q.f.scope, "sqliteq:"), q.f.pages, q.f.hb) if err != nil { t.Fatal(err) } - f := &file{key: q.f.key, db: db, pages: q.f.pages, refs: 1, handles: map[string]*handle{}, recovered: map[string]bool{}} - f.b = &batcher{commit: f.commitRound} - q2, err := newQueue(f, q.name, q.priority, WithRemoveOnComplete(q.removeOnComplete)) + t.Cleanup(func() { f.release() }) + h, err := f.open(q.name, q.priority, []Option{WithRemoveOnComplete(q.removeOnComplete)}) if err != nil { - db.Close() t.Fatal(err) } - t.Cleanup(func() { - q2.Close() - db.Close() - }) - return q2 + return h.queue() } // job is a varmq-style envelope with Job ID id. diff --git a/queues.go b/queues.go index 92c4e62..0d8487f 100644 --- a/queues.go +++ b/queues.go @@ -42,6 +42,10 @@ func New(path string, opts ...StoreOption) (Queues, error) { return nil, fmt.Errorf("sqliteq: checkpoint pages must be at least 1, got %d", cfg.checkpointPages) } + if cfg.heartbeat < minHeartbeat { + return nil, fmt.Errorf("sqliteq: heartbeat timeout must be at least %s, got %s", minHeartbeat, cfg.heartbeat) + } + root, err := layout(path, cfg.singleFile) if err != nil { return nil, err @@ -151,7 +155,7 @@ func (qs *queues) open(name string, priority bool, opts []Option) (*handle, erro fresh := f == nil if fresh { - if f, err = acquire(path, qs.root, qs.cfg.checkpointPages); err != nil { + if f, err = acquire(path, qs.root, qs.cfg.checkpointPages, qs.cfg.heartbeat); err != nil { return nil, err } } diff --git a/registry.go b/registry.go index 9960d49..b083db3 100644 --- a/registry.go +++ b/registry.go @@ -7,6 +7,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/mattn/go-sqlite3" ) @@ -24,6 +25,16 @@ type file struct { rounds atomic.Int64 // rounds committed by b; read by tests handles map[string]*handle // lower-cased queue name -> handle; under registry.mu recovered map[string]bool // lower-cased table -> recovery done; under registry.mu + + // Owner and heartbeat (see owner.go) + owner string // this process's id on the file; on the rows it claims + hb time.Duration // heartbeat timeout + claimed bool // the owner row is written; under mu + lastBeat int64 // Unix ms of the last owner row write; under mu + closing bool // the last release began: no claim, no sweep; under mu + sweeping bool // sweepLoop runs; under registry.mu + stop chan struct{} // closed by the last release + done chan struct{} // closed when sweepLoop returns } // handle is the queue opened under one name in a file; exactly one is set. @@ -82,7 +93,7 @@ var registry = struct { // acquire returns the registry entry for the resolved path, opening the pool // on first use, and takes one reference on it. -func acquire(path, root string, pages int) (*file, error) { +func acquire(path, root string, pages int, hb time.Duration) (*file, error) { registry.mu.Lock() defer registry.mu.Unlock() @@ -93,10 +104,24 @@ func acquire(path, root string, pages int) (*file, error) { if f.pages != pages { return nil, fmt.Errorf("sqliteq: %s is already open with checkpoint pages %d, not %d", path, f.pages, pages) } + if f.hb != hb { + return nil, fmt.Errorf("sqliteq: %s is already open with heartbeat timeout %s, not %s", path, f.hb, hb) + } f.refs++ return f, nil } + f, err := newFile(path, root, pages, hb) + if err != nil { + return nil, err + } + registry.files[path] = f + return f, nil +} + +// newFile opens the pool of a database file under a new owner id, holding +// one reference. It is not in the registry. +func newFile(path, root string, pages int, hb time.Duration) (*file, error) { // sql.Open only validates its arguments; no connection is made here db, err := sql.Open(driverName(pages), dsn(path)) if err != nil { @@ -114,13 +139,17 @@ func acquire(path, root string, pages int) (*file, error) { refs: 1, handles: map[string]*handle{}, recovered: map[string]bool{}, + owner: newOwnerID(), + hb: hb, + stop: make(chan struct{}), + done: make(chan struct{}), } f.b = &batcher{commit: f.commitRound} - registry.files[path] = f return f, nil } -// release drops one reference; the last one closes the pool and the entry. +// release drops one reference; the last one stops the sweep, hands this +// owner's unacknowledged rows back and closes the pool and the entry. func (f *file) release() error { registry.mu.Lock() f.refs-- @@ -132,18 +161,32 @@ func (f *file) release() error { delete(registry.files, f.key) } // Stop the queues' polls before the pool closes under them + tables := make([]string, 0, len(f.handles)) for _, h := range f.handles { h.queue().Close() + tables = append(tables, h.queue().table) } + sweeping := f.sweeping + f.sweeping = false // a repeated release stops nothing twice registry.mu.Unlock() + f.mu.Lock() + f.closing = true + f.mu.Unlock() + if sweeping { + close(f.stop) + <-f.done + } + f.handBack(tables) + // Close waits for running queries, so it runs outside the lock return f.db.Close() } // open returns the handle for name, creating it if there is none or the // existing one was closed. Creation holds registry.mu across table init and -// recovery, so no handle on a table is handed out before its recovery ends. +// the table's first sweep, so no handle on a table is handed out before it +// ends. The first table starts the file's sweep loop. func (f *file) open(name string, priority bool, opts []Option) (*handle, error) { registry.mu.Lock() defer registry.mu.Unlock() @@ -191,12 +234,10 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) h.q = q } - // ponytail: recovery runs on every process's first open, also one that - // never takes jobs (a producer, observer or pauser), and requeues jobs - // another process is running. Upgrade path: owner ids plus heartbeats - // on processing rows, requeueing only rows of dead owners (redisq's model). + // First open: rows of dead owners, and rows an older version left + // without an owner, go back to pending if !f.recovered[key] { - if err := h.queue().requeueNoAck(); err != nil { + if _, err := f.sweep([]*Queue{h.queue()}, true); err != nil { h.queue().Close() // stops a pause poll return nil, fmt.Errorf("failed to requeue unacknowledged rows: %w", err) } @@ -204,6 +245,10 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) } f.handles[key] = h + if !f.sweeping { + f.sweeping = true + go f.sweepLoop() + } return h, nil } diff --git a/workflow.go b/workflow.go index 294823c..c7e2e30 100644 --- a/workflow.go +++ b/workflow.go @@ -162,7 +162,7 @@ func NewWorkflowStore(qs Queues, opts ...WorkflowOption) (*WorkflowStore, error) if closed { return nil, errors.New("sqliteq: queues are closed") } - f, err := acquire(path, q.root, q.cfg.checkpointPages) + f, err := acquire(path, q.root, q.cfg.checkpointPages, q.cfg.heartbeat) if err != nil { return nil, err } diff --git a/workflow_finish.go b/workflow_finish.go index a0c01f1..8f56145 100644 --- a/workflow_finish.go +++ b/workflow_finish.go @@ -293,12 +293,12 @@ func (f *file) purgeStore() (ws *WorkflowStore, done func(), err error) { case err != nil: return nil, none, fmt.Errorf("checking for the workflow file: %w", err) } - if wf, err = acquire(path, root, f.pages); err != nil { + if wf, err = acquire(path, root, f.pages, f.hb); err != nil { return nil, none, fmt.Errorf("opening the workflow file: %w", err) } release = func() { wf.release() } } - qs := &queues{cfg: storeConfig{singleFile: single, checkpointPages: f.pages}, root: root, files: map[*file]struct{}{}} + qs := &queues{cfg: storeConfig{singleFile: single, checkpointPages: f.pages, heartbeat: f.hb}, root: root, files: map[*file]struct{}{}} qs.byID.Store(&map[string]*handle{}) ws = &WorkflowStore{qs: qs, f: wf, single: single, scope: f.scope, cfg: workflowConfig{resultLimit: defaultResultLimit, retention: defaultRetention}} // tables of an older version get the columns this one reads From e5ba952f80cf8510406d5fe7e39fd8299806d446 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 20:23:51 +0600 Subject: [PATCH 03/10] docs: multi-process rules, heartbeat timeout, mixed-version warning --- CHANGELOG.md | 18 ++++++++++++++++++ README.md | 44 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8306082..3c602ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Requires Go 1.27 (was 1.24): `go.mod` requires VarMQ, which needs it. (#22) - Queue name must not contain `__sqliteq_`, in both layouts. (#22) - Reason: a queue's indexes, side tables and triggers use that part in their names. (#22) +- Recovery is by owner and heartbeat, not on open. After a crash, unacknowledged items return one heartbeat timeout (default 30s) after the dead process's last heartbeat, also for a single process that restarts at once. (#28) +- Mixed versions on one file are unsafe: an older process still requeues every processing item on open, and its items carry no owner. Upgrade all processes of a file together. (#28) +- `OnPauseChange` hooks also get `false` when a sweep puts items back into an unpaused queue. (#28) ### Added @@ -87,12 +90,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pause outlives restarts and `Purge`. (#23) - Refused pop sets `Paused`; while set, the queue object polls the row every 50 ms for a remote `Resume`. (#23) +- Owner id per process and file; column `owner` on queue tables, set by a claim, NULL on pending rows. (#28) +- Tables `__sqliteq_owners` (heartbeats) and `__sqliteq_clock` (last sweep), per database file. Index `__sqliteq_processing`. (#28) +- Only a process that claims writes a heartbeat. Producers, inspectors and pausers write none. (#28) +- Sweep on a queue's first open and every sixth of the timeout, in every process with a queue open: requeues processing items whose owner has no live heartbeat. (#28) +- A dead process's items return within one timeout, with no reopen; a bound VarMQ worker is woken. (#28) +- `Close` of the last `Queues` on a file requeues its own unacknowledged items at once and deletes its heartbeat. (#28) +- Clock-gap guard: a gap between sweeps above a third of the timeout, or a clock step back, moves every heartbeat by the gap. (#28) +- `WithHeartbeatTimeout(d)` store option: default 30s, at least 5s. Must match for every `New` in the process on one file. (#28) + ### Changed +- `Acknowledge`, `Reject` and every other finish match the owner: one for an item another process took since returns false. (#28) +- Known limit: a process that froze past the timeout and claimed the same item again can end the second run with the first run's ack. No attempt counter. (#28) - Transactions begin `IMMEDIATE`. (#22) - Finish (`Acknowledge`, `Reject`) always runs in a transaction. (#22) - Pop runs in a transaction while any cancel mark is live. (#22) +### Fixed + +- A second process that opened a queue requeued the jobs another live process was running, so they ran twice. (#27, #28) + ## [0.3.0] - 2026-10-23 ### ⚠️ Breaking Changes diff --git a/README.md b/README.md index a65ddf3..cb8cae5 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ - **Job cancel**: `CancelJob` drops a waiting job, and `OnCancel` tells every process on the file to stop a running one. - **Job records and inspection**: counts, totals and job lists for VarMQ's REST API and vartop. - **[Workflows](#workflows)**: `NewWorkflowStore` keeps VarMQ workflows (a DAG of jobs across queues) in SQLite, so they survive a restart. +- **Recovery by owner**: each process has an owner id and a heartbeat in the file. Only the jobs of a dead process go back to the queue, so processes can share a file. - **Durable queue pause**: `Pause` stops the pops of every process on the file, and the pause outlives restarts. - **Automatic WAL checkpoints**, tunable with `WithCheckpointPages`. - **Thread-safe**: share one queue object across goroutines. @@ -130,7 +131,7 @@ if ok { } ``` -An item taken with `DequeueWithAckId` is marked as processing until it is acknowledged. The first time a queue is opened after its file is opened in the process, every item still marked as processing goes back to pending, so a job that was running when the process stopped runs again. There is no visibility timeout: unacknowledged items return only when the file is next opened, not when a closed queue is opened again on a file that is still open. +An item taken with `DequeueWithAckId` is marked as processing, under the owner id of the process that took it, until it is acknowledged. A process that takes items keeps a heartbeat in the file. When a process dies, its heartbeat runs out after the heartbeat timeout (30 s by default), and any process that has the queue open puts its items back to pending. A clean `Close` of the store puts them back at once. An `Acknowledge` or `Reject` for an item another process has taken since returns false and changes nothing. There is no visibility timeout: a job may run for as long as its process is alive. See [Processes and files](#processes-and-files). `Reject(ackID, reason)` finishes an item that failed for good, as VarMQ does when a job returns an error or panics. It is never requeued. When the job has a job ID, the reason is stored as its outcome error. @@ -360,7 +361,20 @@ Names are case-insensitive in both layouts: `Jobs` and `jobs` are the same queue sqliteq keeps a registry in the process, keyed by the file's real path (made absolute, with symlinks resolved). Every `New` in the process that reaches the same file shares one connection pool and one write lock, so two `New` calls on one path never write through two competing pools. Two `Queues` values can therefore hold the same queue, and you get the same handle from both. The file closes when the last `Queues` using it is closed. -Other processes can open the same file. SQLite's lock orders their writes, and what lives in the file reaches every process: a pause stops every process's pops at once, and a `Resume`, a cancel or a job outcome reaches the others within 50 ms (they poll). Recovery is the limit. Every process that opens a queue runs recovery on its first open of that queue, whatever it does with it: consumers, producers, observers (a process that only serves the REST API) and `Pause`/`Resume` callers alike. Recovery puts every item still marked as processing back to pending, so a process that opens the queue while another one runs its jobs makes those jobs run again. sqliteq does not take an OS lock to stop that. Sharing a file is safe only when every process opens the queue before any of them takes jobs, and no process opens it, or opens it again after a restart, later. +Other processes can open the same file. SQLite's lock orders their writes, and what lives in the file reaches every process: a pause stops every process's pops at once, and a `Resume`, a cancel or a job outcome reaches the others within 50 ms (they poll). + +Recovery is by owner. Each process has a random owner id for each file, and every item it takes carries that id. The rules: + +- **Heartbeat.** A process that takes items keeps a row in `__sqliteq_owners` and renews it every sixth of the heartbeat timeout (`WithHeartbeatTimeout`, 30 s by default, at least 5 s). A process that only adds, inspects or pauses writes no row. +- **Sweep.** Every process with a queue open sweeps that queue on its first open, and then every sixth of the timeout. A sweep puts back to pending the processing items whose owner has no live heartbeat. So a second process never takes the jobs of a live one, whatever it does with the queue. +- **Dead process.** Its items come back about one heartbeat timeout after its last heartbeat, in any process that has the queue open. No reopen is needed. A VarMQ worker bound to the queue is woken. +- **Restart after a crash.** The items of the crashed process also wait for its old heartbeat to run out: about one timeout after the restart, not at open. Lower the timeout for a faster return. +- **Clean exit.** `Close` of the last `Queues` on a file puts the items of this process back to pending at once and deletes its heartbeat. A job that still runs then runs again. +- **Lost item.** An ack or a reject from a process that lost an item returns false. One limit: when a process froze for longer than the timeout and then took the same item again, the ack of its first run ends the second run. sqliteq does not count attempts. +- **Stalls and clock steps.** When no process swept for longer than a third of the timeout (a suspended host), or the clock went back, every heartbeat moves by that gap first. So a stall does not make every process look dead. Liveness uses the wall clock of the one host. +- **Timeout.** A heartbeat renewal can wait up to 5 s behind a write of another process (the busy timeout). Keep the timeout well above that when other processes write to the file. Processes with different timeouts can share a file. +- **One version.** Upgrade every process of a file together. A process of an older version still puts every processing item back when it opens a queue, and its own items carry no owner. +- **No wake across processes.** A worker does not wake when another process adds a job. On macOS, spell a path the same way every time: the registry key keeps the letter case you pass, so `Data` and `data` would be two entries for one folder. On Windows, Go resolves existing folders and files to their on-disk letter case, so different spellings of them share one entry. @@ -370,9 +384,10 @@ On macOS, spell a path the same way every time: the registry key keeps the lette | ----------------------------- | ------------------------------ | ------------- | ---------------------------------------------------------------------- | | `WithSingleFile()` | `New` | folder layout | `path` is one database file holding every queue | | `WithCheckpointPages(n)` | `New` | `1000` | WAL size, in pages, at which SQLite checkpoints automatically | +| `WithHeartbeatTimeout(d)` | `New` | `30s` | Time without a heartbeat after which other processes requeue this process's unacknowledged items; at least `5s` | | `WithRemoveOnComplete(false)` | `NewQueue`, `NewPriorityQueue` | `true` | Keep finished items, marked as `completed`, `failed` or `cancelled`, instead of deleting them | -`New` returns an error instead of panicking: for a path it cannot use, an invalid option (`WithCheckpointPages` below 1, or a nil option), or a path that is an existing file when `WithSingleFile()` is not given. `NewQueue` and `NewPriorityQueue` return an error for a nil option too. +`New` returns an error instead of panicking: for a path it cannot use, an invalid option (`WithCheckpointPages` below 1, `WithHeartbeatTimeout` below 5 s, or a nil option), or a path that is an existing file when `WithSingleFile()` is not given. `NewQueue` and `NewPriorityQueue` return an error for a nil option too. #### Checkpoints @@ -384,7 +399,7 @@ qs, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) Raising it makes those waits rarer, which cuts `Enqueue` tail latency and raises throughput, at the price of a larger WAL on disk: 10000 pages triggers a checkpoint at about 41 MB. -The WAL is a log, not extra stored data: the database holds the same items either way. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. +The WAL is a log, not extra stored data: the database holds the same items either way. Everything in the process that reaches one file must use the same value, and the same heartbeat timeout: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. ## How It Works @@ -394,14 +409,18 @@ Each queue is a table with these columns: - `priority`: priority queues only; a lower number comes out first. Opening an older table as a priority queue adds it. - `data`: the stored item. - `status`: `pending`, `processing`, or on a kept row `completed`, `failed` or `cancelled`. -- `ack`: 0 when an item is added, 1 once it is acknowledged or rejected and kept with `WithRemoveOnComplete(false)` (`status` `completed`, `failed` or `cancelled`). On open, only `processing` rows with `ack` 0 go back to pending. +- `ack`: 0 when an item is added, 1 once it is acknowledged or rejected and kept with `WithRemoveOnComplete(false)` (`status` `completed`, `failed` or `cancelled`). A sweep puts back only `processing` rows with `ack` 0. - `ack_id`: kept from v0.2 so old tables need no rebuild; no longer used. - `job_id`: the id an item was added with through `EnqueueOnce`, unique among the rows that have one; NULL for `Enqueue`. Opening an older table adds it. +- `owner`: the owner id of the process that claimed the item; NULL while pending and on rows of older versions. Opening an older table adds it. - `created_at`, `updated_at`: when the item was added and last changed. On top of that table: -- Two partial indexes: one covers the pending items in queue order. A unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on. +- Three partial indexes: one covers the pending items in queue order. A unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on. One covers the processing items, so a sweep does not read the pending backlog. +- `__sqliteq_owners` holds one heartbeat row per process that claims (`id`, `expires_at`), and `__sqliteq_clock` holds when any process last swept. Both are per database file. +- One sweep goroutine per open file, in every process with a queue open on it. Each sixth of the heartbeat timeout it renews the heartbeat, puts the processing items of dead owners back to pending and deletes expired heartbeats, in one transaction. +- A claim stores the owner id, and a finish matches it (`AND owner = ?`). The first claim of a process, and a claim after a stall of two sweep intervals, writes the heartbeat first. - A claim or a pop is one `UPDATE ... RETURNING` or `DELETE ... RETURNING` statement. It selects nothing while the queue is paused. - `__sqliteq_meta` holds the totals and the pause; a trigger counts each stored add. - `__sqliteq_jobs` holds one outcome record per job ID: `seq`, `job_id`, `state`, `error`, `result` and `finished_at`. `seq` is `AUTOINCREMENT`, so a poll that reads the records above its last `seq` sees every new one. @@ -418,8 +437,11 @@ On top of that table: ## Trade-offs - **At least once, not exactly once.** A job runs again when its process stops before the job is acknowledged. Make handlers idempotent. -- **No visibility timeout.** Unacknowledged items go back to pending only when a process next opens the file. A hung worker keeps its job until then. -- **Sharing a file between processes needs care.** Each process runs recovery on its first open of a queue. So a process that opens a queue late puts the jobs that other processes hold back to pending. See [Processes and files](#processes-and-files). +- **No visibility timeout.** A hung worker keeps its job while its process is alive. Items come back only when their process dies or closes the store. +- **Recovery waits for the heartbeat.** After a crash, the items of the dead process come back one heartbeat timeout later, also for a single process that restarts at once. See [Processes and files](#processes-and-files). +- **A sweep in every process.** Each process with a queue open writes to each open file every sixth of the heartbeat timeout (5 s by default). A claim and an ack also maintain the processing index. +- **A stale ack of the same process counts.** A process that froze past the timeout and took the same item again can end the second run with the ack of the first. +- **One version per file.** Processes of an older version on the same file still requeue the jobs of live processes. - **Polling, not push.** A queue object with watches or `OnCancel` hooks reads the file every 50 ms, in every process. Outcomes, cancels and a `Resume` from another process arrive within about 50 ms. - **Outcomes need the record.** A job without a job ID gets no outcome. A watch that starts after the job's record was trimmed is never called. Pair a watch with a timeout of your own. - **Fixed retention.** The record limits are not configurable. Kept rows (`WithRemoveOnComplete(false)`) have no retention at all. @@ -431,7 +453,7 @@ On top of that table: | Call | Guarantee | If the process stops | | -------------------------------------------- | --------------------------- | ---------------------------------------------------------- | -| `DequeueWithAckId`, `DequeueBatchWithAckIds` | At least once | Unacknowledged items go back to pending on the next open | +| `DequeueWithAckId`, `DequeueBatchWithAckIds` | At least once | Unacknowledged items go back to pending after the heartbeat timeout | | `Dequeue` | At most once | The item is deleted when it is returned | | `EnqueueOnce` | Once per id | Stored once while its row exists; the id survives recovery | | `Enqueue` | Stored when it returns true | Survives a process crash | @@ -447,6 +469,10 @@ sqliteq runs SQLite in WAL mode with `synchronous=NORMAL`, unchanged from v0.2. - Requires Go 1.27, because `go.mod` requires VarMQ. - A queue name must not contain `__sqliteq_`. Rename such a queue before the upgrade, or it fails to open. - The first open of a queue adds the tables `__sqliteq_meta`, `__sqliteq_jobs` and `__sqliteq_cancelled`, and their triggers. The data in the queue is kept. +- The first open of a queue also adds the `owner` column, the index `__sqliteq_processing` and the tables `__sqliteq_owners` and `__sqliteq_clock`. Processing rows of the older version carry no owner and go back to pending on that open, as before. +- Upgrade every process that shares a file at the same time. An older process requeues the jobs of live processes when it opens a queue. +- After a crash, unacknowledged items come back one heartbeat timeout (30 s by default) after the process last renewed its heartbeat, not at the next open. A clean `Close` returns them at once. +- `OnPauseChange` hooks also get `false` when a sweep puts items back into a queue that is not paused. ## Upgrading from v0.2 From 0eaaaa1c7e496be8325ec086c0b33bf91f6d903a Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 20:46:08 +0600 Subject: [PATCH 04/10] fix: shift a heartbeat once per beat, wake on any requeue Review findings: - a dead owner's heartbeat moved on at every open after a gap, so short-lived processes never got its rows; it now moves once until the owner beats again - only the process whose sweep requeued woke its queues; the clock row now counts requeues and each process wakes at its next sweep - a first claim after a clock step back was moved with the old gap; the claim's beat is now a sweep, so the guard runs first - sweep errors are logged, one failing table no longer stops the beat - the sweep loop takes no registry lock - unreadable clock value and NULL owner id no longer stop reaping Refs #27 --- owner.go | 155 +++++++++++++++++++++++++++++++------------------- owner_test.go | 96 +++++++++++++++++++++++++++++-- registry.go | 25 +++++--- 3 files changed, 205 insertions(+), 71 deletions(-) diff --git a/owner.go b/owner.go index 4995d32..612299d 100644 --- a/owner.go +++ b/owner.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "log" "time" ) @@ -19,7 +20,8 @@ import ( // back one timeout after its last beat. // // Times are the host's wall clock in Unix ms. __sqliteq_clock holds when -// any process last swept; see sweep for the gap guard. +// any process last swept (see sweepLocked for the gap guard) and a count of +// the requeues, which tells every process to wake its queues. const ( defaultHeartbeat = 30 * time.Second @@ -30,10 +32,10 @@ const ( processingIndexSuffix = "__sqliteq_processing" - // The time is read by SQLite once the statement holds the write lock, - // so a wait for another process does not shorten the heartbeat + // A beat clears shifted: the owner is alive, so the next stall may + // move its heartbeat on again beatQuery = `INSERT INTO __sqliteq_owners (id, expires_at) VALUES (?, CAST(unixepoch('subsec') * 1000 AS INTEGER) + ?) - ON CONFLICT (id) DO UPDATE SET expires_at = excluded.expires_at` + ON CONFLICT (id) DO UPDATE SET expires_at = excluded.expires_at, shifted = 0` ) func newOwnerID() string { @@ -55,8 +57,10 @@ func (f *file) gapLimit() int64 { return (f.hb / 3).Milliseconds() } // the write mutex. func (f *file) initOwners() error { for _, s := range []string{ - `CREATE TABLE IF NOT EXISTS __sqliteq_owners (id TEXT PRIMARY KEY, expires_at INTEGER NOT NULL)`, - `CREATE TABLE IF NOT EXISTS __sqliteq_clock (id INTEGER PRIMARY KEY CHECK (id = 1), swept_at INTEGER NOT NULL)`, + // shifted: the gap guard moved the heartbeat on since the owner's last beat + `CREATE TABLE IF NOT EXISTS __sqliteq_owners (id TEXT PRIMARY KEY NOT NULL, expires_at INTEGER NOT NULL, shifted INTEGER NOT NULL DEFAULT 0)`, + // moved: requeues by any process, sweep or hand-back + `CREATE TABLE IF NOT EXISTS __sqliteq_clock (id INTEGER PRIMARY KEY CHECK (id = 1), swept_at INTEGER NOT NULL, moved INTEGER NOT NULL DEFAULT 0)`, } { if _, err := f.db.Exec(s); err != nil { return err @@ -87,10 +91,12 @@ func (q *Queue) addOwner() error { return q.f.initOwners() } -// beat writes this process's owner row before a claim when it has none, or -// when its last write is older than two sweep intervals: a process that -// froze must not hand out rows under an owner that already looks dead. The -// caller holds the write mutex. +// beat makes this process's owner row live before a claim when it has +// none, or when its last write is older than two sweep intervals: a process +// that froze must not hand out rows under an owner that already looks dead. +// It is a sweep of no table, so the gap guard runs before the row is +// written and a later sweep does not move a fresh heartbeat. The caller +// holds the write mutex. func (f *file) beat() error { if f.closing { return errClosed @@ -98,34 +104,51 @@ func (f *file) beat() error { if f.claimed && time.Now().UnixMilli()-f.lastBeat <= 2*f.sweepEvery().Milliseconds() { return nil } - if _, err := f.db.Exec(beatQuery, f.owner, f.hb.Milliseconds()); err != nil { + first := !f.claimed + f.claimed = true + if _, err := f.sweepLocked(nil, false); err != nil { + f.claimed = !first return err } - f.claimed, f.lastBeat = true, time.Now().UnixMilli() return nil } -// sweep is one write transaction: it moves the clock, renews this process's -// owner row if it has one, and puts back to pending the unacknowledged -// processing rows of qs whose owner has no live row; with legacy also those -// without an owner, which an older version claimed. It returns the queues -// whose rows moved. -// -// Gap guard: a gap since the last sweep by any process longer than gapLimit -// means no process could beat (a suspended host, a stopped process), and a -// gap below -gapLimit that the clock was set back. Every heartbeat moves on -// by the gap first, so a stall takes no owner for dead at once. With no -// clock row yet, no owner is taken for dead. -func (f *file) sweep(qs []*Queue, legacy bool) (moved []*Queue, err error) { +// sweep runs sweepLocked under the write mutex. wake reports that some +// process requeued rows in the file since this one last looked: its queues +// may have pending rows no local add announced. +func (f *file) sweep(qs []*Queue, legacy bool) (wake bool, err error) { f.mu.Lock() defer f.mu.Unlock() if f.closing { - return nil, errClosed + return false, errClosed + } + moved, err := f.sweepLocked(qs, legacy) + if moved < 0 { + return false, err // nothing committed } + wake = f.seen && moved != f.seenMoved + f.seen, f.seenMoved = true, moved + return wake, err +} +// sweepLocked is one write transaction: it moves the clock, renews this +// process's owner row if it has one, and puts back to pending the +// unacknowledged processing rows of qs whose owner has no live row; with +// legacy also those without an owner, which an older version claimed. It +// returns the file's requeue count, -1 when nothing was committed. A table +// whose requeue fails does not stop the rest; its error is returned. +// +// Gap guard: a gap since the last sweep by any process longer than gapLimit +// means no process could beat (a suspended host, a stopped process), and a +// gap below -gapLimit that the clock was set back. Every heartbeat moves on +// by the gap first, so a stall takes no owner for dead at once. A heartbeat +// moves once between two beats of its owner: a dead owner's time runs out +// even when every process that opens the file is gone before its second +// sweep. With no clock row yet, no owner is taken for dead. +func (f *file) sweepLocked(qs []*Queue, legacy bool) (moved int64, err error) { tx, err := f.db.Begin() if err != nil { - return nil, err + return -1, err } defer tx.Rollback() @@ -133,25 +156,22 @@ func (f *file) sweep(qs []*Queue, legacy bool) (moved []*Queue, err error) { now := time.Now().UnixMilli() var swept int64 reap := true - switch err := tx.QueryRow("SELECT swept_at FROM __sqliteq_clock").Scan(&swept); { + // CAST: a value of another type must not fail every sweep for good + switch err := tx.QueryRow("SELECT CAST(swept_at AS INTEGER), CAST(moved AS INTEGER) FROM __sqliteq_clock").Scan(&swept, &moved); { case errors.Is(err, sql.ErrNoRows): reap = false case err != nil: - return nil, err + return -1, err default: if gap := now - swept; gap > f.gapLimit() || gap < -f.gapLimit() { - if _, err := tx.Exec("UPDATE __sqliteq_owners SET expires_at = expires_at + ?", gap); err != nil { - return nil, err + if _, err := tx.Exec("UPDATE __sqliteq_owners SET expires_at = expires_at + ?, shifted = 1 WHERE shifted = 0", gap); err != nil { + return -1, err } } } - if _, err := tx.Exec(`INSERT INTO __sqliteq_clock (id, swept_at) VALUES (1, ?) - ON CONFLICT (id) DO UPDATE SET swept_at = excluded.swept_at`, now); err != nil { - return nil, err - } if f.claimed { if _, err := tx.Exec(beatQuery, f.owner, f.hb.Milliseconds()); err != nil { - return nil, err + return -1, err } } @@ -167,38 +187,49 @@ func (f *file) sweep(qs []*Queue, legacy bool) (moved []*Queue, err error) { case legacy: where = "owner IS NULL" } + var errs []error if where != "" { - at := time.UnixMilli(now).UTC() + at, some := time.UnixMilli(now).UTC(), false for _, q := range qs { res, err := tx.Exec(fmt.Sprintf("UPDATE %s SET status = 'pending', owner = NULL, updated_at = ? WHERE status = 'processing' AND ack = 0 AND %s", q.table, where), append([]any{at}, args...)...) if err != nil { - return nil, err + errs = append(errs, fmt.Errorf("%s: %w", q.name, err)) + continue } if n, _ := res.RowsAffected(); n > 0 { - moved = append(moved, q) + some = true } } + if some { + moved++ + } + } + if _, err := tx.Exec(`INSERT INTO __sqliteq_clock (id, swept_at, moved) VALUES (1, ?, ?) + ON CONFLICT (id) DO UPDATE SET swept_at = excluded.swept_at, moved = excluded.moved`, now, moved); err != nil { + return -1, err } if reap { if _, err := tx.Exec("DELETE FROM __sqliteq_owners WHERE expires_at <= ?", now); err != nil { - return nil, err + return -1, err } } if err := tx.Commit(); err != nil { - return nil, err + return -1, err } if f.claimed { f.lastBeat = now } - return moved, nil + return moved, errors.Join(errs...) } // sweepLoop sweeps every table the process has open on the file, every -// sweepEvery until the last release, and wakes the queues whose rows moved. -// It runs in every process with a queue open, also one that never claims: -// VarMQ pops only when Len is above 0, so a consumer that waits on an empty -// queue would never start it, and a dead owner's rows would stay. +// sweepEvery until the last release, and wakes its queues when any process +// requeued rows. It runs in every process with a queue open, also one that +// never claims: VarMQ pops only when Len is above 0, so a consumer that +// waits on an empty queue would never start it, and a dead owner's rows +// would stay. It takes no registry lock: an open of another file that waits +// for SQLite must not hold up this file's heartbeat. func (f *file) sweepLoop() { defer close(f.done) t := time.NewTicker(f.sweepEvery()) @@ -210,17 +241,16 @@ func (f *file) sweepLoop() { case <-t.C: } - registry.mu.Lock() - qs := make([]*Queue, 0, len(f.handles)) - for _, h := range f.handles { - qs = append(qs, h.queue()) - } - registry.mu.Unlock() - + qs := *f.swept.Load() // A failed sweep (a busy file) is tried again at the next tick - moved, _ := f.sweep(qs, false) - for _, q := range moved { - q.wake() + wake, err := f.sweep(qs, false) + if err != nil && !errors.Is(err, errClosed) { + log.Printf("sqliteq: sweep of %s: %v", f.key, err) + } + if wake { + for _, q := range qs { + q.wake() + } } } } @@ -239,10 +269,19 @@ func (f *file) handBack(tables []string) { return } defer tx.Rollback() - now := time.Now().UTC() + now, some := time.Now().UTC(), false for _, t := range tables { - if _, err := tx.Exec(fmt.Sprintf("UPDATE %s SET status = 'pending', owner = NULL, updated_at = ? WHERE status = 'processing' AND ack = 0 AND owner = ?", t), - now, f.owner); err != nil { + res, err := tx.Exec(fmt.Sprintf("UPDATE %s SET status = 'pending', owner = NULL, updated_at = ? WHERE status = 'processing' AND ack = 0 AND owner = ?", t), + now, f.owner) + if err != nil { + return + } + if n, _ := res.RowsAffected(); n > 0 { + some = true + } + } + if some { // the other processes wake their queues + if _, err := tx.Exec("UPDATE __sqliteq_clock SET moved = moved + 1"); err != nil { return } } diff --git a/owner_test.go b/owner_test.go index 52acf75..eb3b03f 100644 --- a/owner_test.go +++ b/owner_test.go @@ -202,6 +202,73 @@ func TestOwnerDead(t *testing.T) { }) } +// Processes that open the queue and are gone before their second sweep +// still get a dead owner's row back: its heartbeat moves on once only. +func TestOwnerDeadShortLived(t *testing.T) { + const hb = 600 * time.Millisecond + qs := fastStore(t, t.TempDir(), hb) + a, err := qs.NewQueue("q") + if err != nil { + t.Fatal(err) + } + claim1(t, a) + kill(a.f) + for i := 0; ; i++ { + time.Sleep(hb / 2) // above the gap limit + f, err := newFile(a.f.key, strings.TrimPrefix(a.f.scope, "sqliteq:"), a.f.pages, hb) + if err != nil { + t.Fatal(err) + } + h, err := f.open("q", false, nil) + if err != nil { + t.Fatal(err) + } + n := h.q.Len() + f.release() + if n == 1 { + return + } + if i == 10 { + t.Fatal("Expected the dead owner's row back within a few opens") + } + } +} + +// A requeue by another process's sweep, or by its hand-back, is seen by +// this process's next sweep, which wakes its queues. +func TestWakeAfterOthersRequeue(t *testing.T) { + a, qs := openQueue(t) + defer qs.Close() + sweep := func(q *Queue) bool { + t.Helper() + wake, err := q.f.sweep([]*Queue{q}, false) + if err != nil { + t.Fatal(err) + } + return wake + } + if sweep(a) { + t.Fatal("Expected no wake with nothing requeued") + } + observer, dead := otherProcess(t, a), otherProcess(t, a) + claim1(t, dead) + kill(dead.f) + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = 0") + sweep(observer) // requeues + if a.Len() != 1 || !sweep(a) || sweep(a) { + t.Fatal("Expected one wake after the observer's requeue") + } + + leaver := otherProcess(t, a) + if _, ok, _ := leaver.DequeueWithAckId(); !ok { + t.Fatal("claim failed") + } + leaver.f.release() + if a.Len() != 1 || !sweep(a) { + t.Fatal("Expected a wake after another process's hand-back") + } +} + // After a crash the restarted process gets its old rows back one timeout // later, not at open. func TestOwnerRestart(t *testing.T) { @@ -336,8 +403,10 @@ func TestClockGap(t *testing.T) { if status, expires := sweep(t, a); status != "processing" || expires < now { t.Fatalf("Expected the owner kept after a long gap, got %s, expires %d ms from now", status, expires-now) } - // Its second left runs out as usual - mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = expires_at - 2000") + // A second long gap with no beat of the owner between moves nothing + now = time.Now().UnixMilli() + mustExec(t, a.f.db, "UPDATE __sqliteq_clock SET swept_at = ?", now-hour) + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = ?", now-1) if status, _ := sweep(t, a); status != "pending" || ownerRows(t, a.f.db) != 0 { t.Fatalf("Expected the dead owner reaped, got %s, %d owner rows", status, ownerRows(t, a.f.db)) } @@ -352,6 +421,23 @@ func TestClockGap(t *testing.T) { t.Fatalf("Expected the heartbeat moved back with the clock, got %s, expires %d ms from now", status, expires-now) } }) + // A first claim right after the clock went back is not moved with it + t.Run("claim after clock set back", func(t *testing.T) { + a, _ := openQueue(t) + b := otherProcess(t, a) + mustExec(t, a.f.db, "UPDATE __sqliteq_clock SET swept_at = ?", time.Now().UnixMilli()+hour) + ack := claim1(t, b) + if status, _ := sweep(t, a); status != "processing" || !b.Acknowledge(ack) { + t.Fatalf("Expected the fresh owner kept, got %s", status) + } + }) + t.Run("unreadable clock", func(t *testing.T) { + a, _ := setup(t) + mustExec(t, a.f.db, "UPDATE __sqliteq_clock SET swept_at = 'x'") + if _, err := a.f.sweep([]*Queue{a}, false); err != nil { + t.Fatalf("Expected the sweep to repair the clock, got %v", err) + } + }) t.Run("no clock row", func(t *testing.T) { a, _ := setup(t) mustExec(t, a.f.db, "DELETE FROM __sqliteq_clock") @@ -460,7 +546,7 @@ func TestHelperProcess(t *testing.T) { t.Skip("helper of TestKilledProcess") } qs := mustNew(t, dir) - qs.(*queues).cfg.heartbeat = time.Second + qs.(*queues).cfg.heartbeat = 2 * time.Second q, err := qs.NewQueue("q") if err != nil { t.Fatal(err) @@ -478,7 +564,7 @@ func TestHelperProcess(t *testing.T) { // opened first and never claimed, with no reopen. func TestKilledProcess(t *testing.T) { dir := t.TempDir() - qs := fastStore(t, dir, time.Second) + qs := fastStore(t, dir, 2*time.Second) q, err := qs.NewQueue("q") if err != nil { t.Fatal(err) @@ -524,7 +610,7 @@ func TestKilledProcess(t *testing.T) { select { case v := <-ran: t.Fatalf("job %d ran while its owner is alive", v) - case <-time.After(2500 * time.Millisecond): // well past one timeout + case <-time.After(3 * time.Second): // past one timeout } if status, _ := rowOf(t, q.f.db); status != "processing" { t.Fatalf("Expected the live owner's row processing, got %s", status) diff --git a/registry.go b/registry.go index b083db3..2f95e4e 100644 --- a/registry.go +++ b/registry.go @@ -27,14 +27,17 @@ type file struct { recovered map[string]bool // lower-cased table -> recovery done; under registry.mu // Owner and heartbeat (see owner.go) - owner string // this process's id on the file; on the rows it claims - hb time.Duration // heartbeat timeout - claimed bool // the owner row is written; under mu - lastBeat int64 // Unix ms of the last owner row write; under mu - closing bool // the last release began: no claim, no sweep; under mu - sweeping bool // sweepLoop runs; under registry.mu - stop chan struct{} // closed by the last release - done chan struct{} // closed when sweepLoop returns + owner string // this process's id on the file; on the rows it claims + hb time.Duration // heartbeat timeout + claimed bool // the owner row is written; under mu + lastBeat int64 // Unix ms of the last owner row write; under mu + closing bool // the last release began: no claim, no sweep; under mu + sweeping bool // sweepLoop runs; under registry.mu + swept atomic.Pointer[[]*Queue] // the handles' queues, for sweepLoop; stored under registry.mu + seen bool // seenMoved is set; under mu + seenMoved int64 // the file's requeue count at the last sweep; under mu + stop chan struct{} // closed by the last release + done chan struct{} // closed when sweepLoop returns } // handle is the queue opened under one name in a file; exactly one is set. @@ -145,6 +148,7 @@ func newFile(path, root string, pages int, hb time.Duration) (*file, error) { done: make(chan struct{}), } f.b = &batcher{commit: f.commitRound} + f.swept.Store(&[]*Queue{}) return f, nil } @@ -245,6 +249,11 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) } f.handles[key] = h + qs := make([]*Queue, 0, len(f.handles)) + for _, h := range f.handles { + qs = append(qs, h.queue()) + } + f.swept.Store(&qs) if !f.sweeping { f.sweeping = true go f.sweepLoop() From 28ef3bedd043a61d5e87aec1988dc0b86efb665d Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 20:46:08 +0600 Subject: [PATCH 05/10] docs: requeue wake, single shift, file close with workflow store --- CHANGELOG.md | 7 ++++--- README.md | 12 ++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c602ab..2b86cd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,9 +94,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Tables `__sqliteq_owners` (heartbeats) and `__sqliteq_clock` (last sweep), per database file. Index `__sqliteq_processing`. (#28) - Only a process that claims writes a heartbeat. Producers, inspectors and pausers write none. (#28) - Sweep on a queue's first open and every sixth of the timeout, in every process with a queue open: requeues processing items whose owner has no live heartbeat. (#28) -- A dead process's items return within one timeout, with no reopen; a bound VarMQ worker is woken. (#28) -- `Close` of the last `Queues` on a file requeues its own unacknowledged items at once and deletes its heartbeat. (#28) -- Clock-gap guard: a gap between sweeps above a third of the timeout, or a clock step back, moves every heartbeat by the gap. (#28) +- A dead process's items return about one timeout after its last heartbeat, with no reopen. (#28) +- Requeue count in `__sqliteq_clock`: each process wakes its bound VarMQ workers at its next sweep, whichever process requeued. (#28) +- Last close of a file in the process (last `Queues`, and the `WorkflowStore` under `WithSingleFile`) requeues its own unacknowledged items at once and deletes its heartbeat. (#28) +- Clock-gap guard: a gap between sweeps above a third of the timeout, or a clock step back, moves every heartbeat by the gap, once per heartbeat until its owner renews it. (#28) - `WithHeartbeatTimeout(d)` store option: default 30s, at least 5s. Must match for every `New` in the process on one file. (#28) ### Changed diff --git a/README.md b/README.md index cb8cae5..33f9a26 100644 --- a/README.md +++ b/README.md @@ -367,12 +367,12 @@ Recovery is by owner. Each process has a random owner id for each file, and ever - **Heartbeat.** A process that takes items keeps a row in `__sqliteq_owners` and renews it every sixth of the heartbeat timeout (`WithHeartbeatTimeout`, 30 s by default, at least 5 s). A process that only adds, inspects or pauses writes no row. - **Sweep.** Every process with a queue open sweeps that queue on its first open, and then every sixth of the timeout. A sweep puts back to pending the processing items whose owner has no live heartbeat. So a second process never takes the jobs of a live one, whatever it does with the queue. -- **Dead process.** Its items come back about one heartbeat timeout after its last heartbeat, in any process that has the queue open. No reopen is needed. A VarMQ worker bound to the queue is woken. +- **Dead process.** Its items come back about one heartbeat timeout after its last heartbeat, in any process that has the queue open. No reopen is needed. Each process sees the requeue at its next sweep and wakes the VarMQ workers bound to its queues, whichever process did the requeue. - **Restart after a crash.** The items of the crashed process also wait for its old heartbeat to run out: about one timeout after the restart, not at open. Lower the timeout for a faster return. -- **Clean exit.** `Close` of the last `Queues` on a file puts the items of this process back to pending at once and deletes its heartbeat. A job that still runs then runs again. +- **Clean exit.** When the process closes a file, it puts its own items back to pending at once and deletes its heartbeat. A file closes with the last `Queues` that holds it, and under `WithSingleFile` with the `WorkflowStore` too. A job that still runs then runs again. - **Lost item.** An ack or a reject from a process that lost an item returns false. One limit: when a process froze for longer than the timeout and then took the same item again, the ack of its first run ends the second run. sqliteq does not count attempts. -- **Stalls and clock steps.** When no process swept for longer than a third of the timeout (a suspended host), or the clock went back, every heartbeat moves by that gap first. So a stall does not make every process look dead. Liveness uses the wall clock of the one host. -- **Timeout.** A heartbeat renewal can wait up to 5 s behind a write of another process (the busy timeout). Keep the timeout well above that when other processes write to the file. Processes with different timeouts can share a file. +- **Stalls and clock steps.** When no process swept for longer than a third of the timeout (a suspended host), or the clock went back, every heartbeat moves by that gap first. So a stall does not make every process look dead. A heartbeat moves once only until its process renews it, so the items of a dead process still come back when every later process is short-lived. Liveness uses the wall clock of the one host. +- **Timeout.** A heartbeat renewal can wait up to 5 s behind a write of another process (the busy timeout). Keep the timeout well above that when other processes write to the file. Processes with different timeouts can share a file; a process with a short timeout that opens the file can delay the return of a dead process's items by one sweep gap. - **One version.** Upgrade every process of a file together. A process of an older version still puts every processing item back when it opens a queue, and its own items carry no owner. - **No wake across processes.** A worker does not wake when another process adds a job. @@ -418,9 +418,9 @@ Each queue is a table with these columns: On top of that table: - Three partial indexes: one covers the pending items in queue order. A unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on. One covers the processing items, so a sweep does not read the pending backlog. -- `__sqliteq_owners` holds one heartbeat row per process that claims (`id`, `expires_at`), and `__sqliteq_clock` holds when any process last swept. Both are per database file. +- `__sqliteq_owners` holds one heartbeat row per process that claims (`id`, `expires_at`, `shifted`). `__sqliteq_clock` holds when any process last swept and a count of the requeues. Both are per database file. - One sweep goroutine per open file, in every process with a queue open on it. Each sixth of the heartbeat timeout it renews the heartbeat, puts the processing items of dead owners back to pending and deletes expired heartbeats, in one transaction. -- A claim stores the owner id, and a finish matches it (`AND owner = ?`). The first claim of a process, and a claim after a stall of two sweep intervals, writes the heartbeat first. +- A claim stores the owner id, and a finish matches it (`AND owner = ?`). The first claim of a process, and a claim after a stall of two sweep intervals, runs a sweep transaction first, which writes the heartbeat. - A claim or a pop is one `UPDATE ... RETURNING` or `DELETE ... RETURNING` statement. It selects nothing while the queue is paused. - `__sqliteq_meta` holds the totals and the pause; a trigger counts each stored add. - `__sqliteq_jobs` holds one outcome record per job ID: `seq`, `job_id`, `state`, `error`, `result` and `finished_at`. `seq` is `AUTOINCREMENT`, so a poll that reads the records above its last `seq` sees every new one. From ada8e5a134a9c58196d9213c6d136d609fea554f Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 20:49:40 +0600 Subject: [PATCH 06/10] fix: a first-open sweep keeps the requeue count for the loop --- owner.go | 5 +++++ owner_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/owner.go b/owner.go index 612299d..6f55db7 100644 --- a/owner.go +++ b/owner.go @@ -126,6 +126,11 @@ func (f *file) sweep(qs []*Queue, legacy bool) (wake bool, err error) { if moved < 0 { return false, err // nothing committed } + // A first open has no one to wake for its own table, and must not use + // up a requeue the loop has not seen: the next tick wakes for it + if legacy && f.seen { + return false, err + } wake = f.seen && moved != f.seenMoved f.seen, f.seenMoved = true, moved return wake, err diff --git a/owner_test.go b/owner_test.go index eb3b03f..f4478c6 100644 --- a/owner_test.go +++ b/owner_test.go @@ -269,6 +269,30 @@ func TestWakeAfterOthersRequeue(t *testing.T) { } } +// Opening another queue of the file does not use up a requeue the sweep +// loop has not seen. +func TestOpenKeepsWake(t *testing.T) { + qs := mustNew(t, filepath.Join(t.TempDir(), "one.db"), WithSingleFile()) + defer qs.Close() + a, err := qs.NewQueue("q") + if err != nil { + t.Fatal(err) + } + observer, dead := otherProcess(t, a), otherProcess(t, a) + claim1(t, dead) + kill(dead.f) + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = 0") + if _, err := observer.f.sweep([]*Queue{observer}, false); err != nil { + t.Fatal(err) + } + if _, err := qs.NewQueue("y"); err != nil { + t.Fatal(err) + } + if wake, err := a.f.sweep([]*Queue{a}, false); err != nil || !wake { + t.Fatalf("Expected the next sweep to wake after an open in between, got %t %v", wake, err) + } +} + // After a crash the restarted process gets its old rows back one timeout // later, not at open. func TestOwnerRestart(t *testing.T) { From d96785a940ac8cba1e1fbd8cb0a36b498ea3959c Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 20:52:52 +0600 Subject: [PATCH 07/10] docs: note the second-stall limit of the gap guard --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33f9a26..863441f 100644 --- a/README.md +++ b/README.md @@ -371,7 +371,7 @@ Recovery is by owner. Each process has a random owner id for each file, and ever - **Restart after a crash.** The items of the crashed process also wait for its old heartbeat to run out: about one timeout after the restart, not at open. Lower the timeout for a faster return. - **Clean exit.** When the process closes a file, it puts its own items back to pending at once and deletes its heartbeat. A file closes with the last `Queues` that holds it, and under `WithSingleFile` with the `WorkflowStore` too. A job that still runs then runs again. - **Lost item.** An ack or a reject from a process that lost an item returns false. One limit: when a process froze for longer than the timeout and then took the same item again, the ack of its first run ends the second run. sqliteq does not count attempts. -- **Stalls and clock steps.** When no process swept for longer than a third of the timeout (a suspended host), or the clock went back, every heartbeat moves by that gap first. So a stall does not make every process look dead. A heartbeat moves once only until its process renews it, so the items of a dead process still come back when every later process is short-lived. Liveness uses the wall clock of the one host. +- **Stalls and clock steps.** When no process swept for longer than a third of the timeout (a suspended host), or the clock went back, every heartbeat moves by that gap first. So a stall does not make every process look dead. A heartbeat moves once only until its process renews it, so the items of a dead process still come back when every later process is short-lived. A second stall before that renewal can make a live process look dead, and its jobs then run twice. Liveness uses the wall clock of the one host. - **Timeout.** A heartbeat renewal can wait up to 5 s behind a write of another process (the busy timeout). Keep the timeout well above that when other processes write to the file. Processes with different timeouts can share a file; a process with a short timeout that opens the file can delay the return of a dead process's items by one sweep gap. - **One version.** Upgrade every process of a file together. A process of an older version still puts every processing item back when it opens a queue, and its own items carry no owner. - **No wake across processes.** A worker does not wake when another process adds a job. From 464bae472c6003d70583da37b7a890aecbb781f6 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 21:02:33 +0600 Subject: [PATCH 08/10] perf: drop the processing index, scan tables only when an owner dies A claim and an ack maintained the index on every call (+8% ack, +16% batch pop). A sweep now reads a queue table on its first open, when it drops an expired owner row, or when the file's count shows another process dropped one. The count moves on each drop, so every process reads its open tables once per death. Refs #27 --- CHANGELOG.md | 3 +- README.md | 10 +++--- migration_test.go | 2 +- owner.go | 81 +++++++++++++++++++++++++++-------------------- owner_test.go | 49 +++++++++++++++++++++------- 5 files changed, 92 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b86cd1..7f57371 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,7 +91,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Refused pop sets `Paused`; while set, the queue object polls the row every 50 ms for a remote `Resume`. (#23) - Owner id per process and file; column `owner` on queue tables, set by a claim, NULL on pending rows. (#28) -- Tables `__sqliteq_owners` (heartbeats) and `__sqliteq_clock` (last sweep), per database file. Index `__sqliteq_processing`. (#28) +- Tables `__sqliteq_owners` (heartbeats) and `__sqliteq_clock` (last sweep), per database file. (#28) +- No index on processing rows: a sweep scans a queue table only on its first open and when a heartbeat expired, so a claim and an ack maintain nothing new. (#28) - Only a process that claims writes a heartbeat. Producers, inspectors and pausers write none. (#28) - Sweep on a queue's first open and every sixth of the timeout, in every process with a queue open: requeues processing items whose owner has no live heartbeat. (#28) - A dead process's items return about one timeout after its last heartbeat, with no reopen. (#28) diff --git a/README.md b/README.md index 863441f..bab3a8f 100644 --- a/README.md +++ b/README.md @@ -366,7 +366,7 @@ Other processes can open the same file. SQLite's lock orders their writes, and w Recovery is by owner. Each process has a random owner id for each file, and every item it takes carries that id. The rules: - **Heartbeat.** A process that takes items keeps a row in `__sqliteq_owners` and renews it every sixth of the heartbeat timeout (`WithHeartbeatTimeout`, 30 s by default, at least 5 s). A process that only adds, inspects or pauses writes no row. -- **Sweep.** Every process with a queue open sweeps that queue on its first open, and then every sixth of the timeout. A sweep puts back to pending the processing items whose owner has no live heartbeat. So a second process never takes the jobs of a live one, whatever it does with the queue. +- **Sweep.** Every process with a queue open sweeps that queue on its first open, and then every sixth of the timeout. A sweep deletes the heartbeats that ran out and, when it found one, puts back to pending the processing items whose owner has no live heartbeat. So a second process never takes the jobs of a live one, whatever it does with the queue. - **Dead process.** Its items come back about one heartbeat timeout after its last heartbeat, in any process that has the queue open. No reopen is needed. Each process sees the requeue at its next sweep and wakes the VarMQ workers bound to its queues, whichever process did the requeue. - **Restart after a crash.** The items of the crashed process also wait for its old heartbeat to run out: about one timeout after the restart, not at open. Lower the timeout for a faster return. - **Clean exit.** When the process closes a file, it puts its own items back to pending at once and deletes its heartbeat. A file closes with the last `Queues` that holds it, and under `WithSingleFile` with the `WorkflowStore` too. A job that still runs then runs again. @@ -417,9 +417,9 @@ Each queue is a table with these columns: On top of that table: -- Three partial indexes: one covers the pending items in queue order. A unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on. One covers the processing items, so a sweep does not read the pending backlog. +- Two partial indexes: one covers the pending items in queue order. A unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on. Processing items have no index, so a claim and an ack do not pay for one. - `__sqliteq_owners` holds one heartbeat row per process that claims (`id`, `expires_at`, `shifted`). `__sqliteq_clock` holds when any process last swept and a count of the requeues. Both are per database file. -- One sweep goroutine per open file, in every process with a queue open on it. Each sixth of the heartbeat timeout it renews the heartbeat, puts the processing items of dead owners back to pending and deletes expired heartbeats, in one transaction. +- One sweep goroutine per open file, in every process with a queue open on it. Each sixth of the heartbeat timeout it renews the heartbeat and deletes expired heartbeats, in one transaction on the two small tables. It reads a queue table, one scan, only when a heartbeat expired: the process that deletes it moves a count in `__sqliteq_clock`, and each process that sees the count move puts the processing items of dead owners in its open queues back to pending. The first open of a queue reads its table too, as recovery always did. - A claim stores the owner id, and a finish matches it (`AND owner = ?`). The first claim of a process, and a claim after a stall of two sweep intervals, runs a sweep transaction first, which writes the heartbeat. - A claim or a pop is one `UPDATE ... RETURNING` or `DELETE ... RETURNING` statement. It selects nothing while the queue is paused. - `__sqliteq_meta` holds the totals and the pause; a trigger counts each stored add. @@ -439,7 +439,7 @@ On top of that table: - **At least once, not exactly once.** A job runs again when its process stops before the job is acknowledged. Make handlers idempotent. - **No visibility timeout.** A hung worker keeps its job while its process is alive. Items come back only when their process dies or closes the store. - **Recovery waits for the heartbeat.** After a crash, the items of the dead process come back one heartbeat timeout later, also for a single process that restarts at once. See [Processes and files](#processes-and-files). -- **A sweep in every process.** Each process with a queue open writes to each open file every sixth of the heartbeat timeout (5 s by default). A claim and an ack also maintain the processing index. +- **A sweep in every process.** Each process with a queue open writes to each open file every sixth of the heartbeat timeout (5 s by default). When a process dies, each live process scans its open queue tables once, under the write lock. - **A stale ack of the same process counts.** A process that froze past the timeout and took the same item again can end the second run with the ack of the first. - **One version per file.** Processes of an older version on the same file still requeue the jobs of live processes. - **Polling, not push.** A queue object with watches or `OnCancel` hooks reads the file every 50 ms, in every process. Outcomes, cancels and a `Resume` from another process arrive within about 50 ms. @@ -469,7 +469,7 @@ sqliteq runs SQLite in WAL mode with `synchronous=NORMAL`, unchanged from v0.2. - Requires Go 1.27, because `go.mod` requires VarMQ. - A queue name must not contain `__sqliteq_`. Rename such a queue before the upgrade, or it fails to open. - The first open of a queue adds the tables `__sqliteq_meta`, `__sqliteq_jobs` and `__sqliteq_cancelled`, and their triggers. The data in the queue is kept. -- The first open of a queue also adds the `owner` column, the index `__sqliteq_processing` and the tables `__sqliteq_owners` and `__sqliteq_clock`. Processing rows of the older version carry no owner and go back to pending on that open, as before. +- The first open of a queue also adds the `owner` column and the tables `__sqliteq_owners` and `__sqliteq_clock`. Processing rows of the older version carry no owner and go back to pending on that open, as before. - Upgrade every process that shares a file at the same time. An older process requeues the jobs of live processes when it opens a queue. - After a crash, unacknowledged items come back one heartbeat timeout (30 s by default) after the process last renewed its heartbeat, not at the next open. A clean `Close` returns them at once. - `OnPauseChange` hooks also get `false` when a sweep puts items back into a queue that is not paused. diff --git a/migration_test.go b/migration_test.go index 8332fd7..ee2a3ec 100644 --- a/migration_test.go +++ b/migration_test.go @@ -129,7 +129,7 @@ func TestMigrateV02File(t *testing.T) { dequeue, length, db = q.Dequeue, q.Len, q.f.db } - want := []string{name + jobIDIndexSuffix, name + pendingIndexSuffix, name + processingIndexSuffix, "sqlite_autoindex_" + name + "_1"} + want := []string{name + jobIDIndexSuffix, name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { t.Errorf("Round %d: indexes = %v, want %v", round, got, want) } diff --git a/owner.go b/owner.go index 6f55db7..9bcdf59 100644 --- a/owner.go +++ b/owner.go @@ -14,14 +14,20 @@ import ( // has a random owner id, stored on every row it claims. An owner that // claims keeps a row in __sqliteq_owners, renewed by the sweep; a process // that never claims writes none. The sweep runs on a table's first open and -// then every hb/6 for every table the process has open: it requeues the +// then every hb/6: it drops the owner rows that ran out and requeues the // processing rows whose owner has no live row. SQLite has no presence // signal, so the heartbeat is the only liveness: a killed owner's rows come // back one timeout after its last beat. // -// Times are the host's wall clock in Unix ms. __sqliteq_clock holds when -// any process last swept (see sweepLocked for the gap guard) and a count of -// the requeues, which tells every process to wake its queues. +// Queue tables have no index on processing rows: a claim and an ack would +// pay for it on every call. So a sweep reads a queue table only on its +// first open, as recovery always did, and when an owner died: the sweep +// that drops a dead owner's row moves the count in __sqliteq_clock, and +// every process that sees the count move reads its open tables once and +// wakes its queues. +// +// Times are the host's wall clock in Unix ms. __sqliteq_clock also holds +// when any process last swept (see sweepLocked for the gap guard). const ( defaultHeartbeat = 30 * time.Second @@ -30,8 +36,6 @@ const ( // writers in other processes the timeout must stay well above 5s. minHeartbeat = 5 * time.Second - processingIndexSuffix = "__sqliteq_processing" - // A beat clears shifted: the owner is alive, so the next stall may // move its heartbeat on again beatQuery = `INSERT INTO __sqliteq_owners (id, expires_at) VALUES (?, CAST(unixepoch('subsec') * 1000 AS INTEGER) + ?) @@ -59,7 +63,7 @@ func (f *file) initOwners() error { for _, s := range []string{ // shifted: the gap guard moved the heartbeat on since the owner's last beat `CREATE TABLE IF NOT EXISTS __sqliteq_owners (id TEXT PRIMARY KEY NOT NULL, expires_at INTEGER NOT NULL, shifted INTEGER NOT NULL DEFAULT 0)`, - // moved: requeues by any process, sweep or hand-back + // moved: counts the dead owners dropped and the requeues, by any process `CREATE TABLE IF NOT EXISTS __sqliteq_clock (id INTEGER PRIMARY KEY CHECK (id = 1), swept_at INTEGER NOT NULL, moved INTEGER NOT NULL DEFAULT 0)`, } { if _, err := f.db.Exec(s); err != nil { @@ -69,9 +73,8 @@ func (f *file) initOwners() error { return nil } -// addOwner adds the owner column a claim fills, when it is missing, the -// index the sweep reads, and the file's owner tables. Pending rows and rows -// of older versions are NULL. +// addOwner adds the owner column a claim fills, when it is missing, and the +// file's owner tables. Pending rows and rows of older versions are NULL. func (q *Queue) addOwner() error { exists, err := hasColumn(q.f.db, q.name, "owner") if err != nil { @@ -82,12 +85,6 @@ func (q *Queue) addOwner() error { return err } } - // Without it a sweep scans the whole table, pending backlog included. - // Keyed by id: the smallest entry for a claim and an ack to maintain. - if _, err := q.f.db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (id) WHERE status = 'processing'", - quoteIdent(q.name+processingIndexSuffix), q.table)); err != nil { - return err - } return q.f.initOwners() } @@ -106,7 +103,7 @@ func (f *file) beat() error { } first := !f.claimed f.claimed = true - if _, err := f.sweepLocked(nil, false); err != nil { + if _, err := f.sweepLocked(nil, false, false); err != nil { f.claimed = !first return err } @@ -122,26 +119,33 @@ func (f *file) sweep(qs []*Queue, legacy bool) (wake bool, err error) { if f.closing { return false, errClosed } - moved, err := f.sweepLocked(qs, legacy) + moved, err := f.sweepLocked(qs, legacy, f.seen) if moved < 0 { return false, err // nothing committed } // A first open has no one to wake for its own table, and must not use - // up a requeue the loop has not seen: the next tick wakes for it + // up a count the loop has not seen: the next tick reads the other + // tables and wakes for it if legacy && f.seen { return false, err } wake = f.seen && moved != f.seenMoved - f.seen, f.seenMoved = true, moved + if err == nil { // a table that failed is read again at the next tick + f.seen, f.seenMoved = true, moved + } return wake, err } // sweepLocked is one write transaction: it moves the clock, renews this -// process's owner row if it has one, and puts back to pending the -// unacknowledged processing rows of qs whose owner has no live row; with -// legacy also those without an owner, which an older version claimed. It -// returns the file's requeue count, -1 when nothing was committed. A table -// whose requeue fails does not stop the rest; its error is returned. +// process's owner row if it has one and drops the owner rows that ran out. +// It reads the tables of qs, a scan each, only when it has to: on a first +// open (legacy), when it dropped an owner row, or when track is set and the +// file's count moved since f.seenMoved (another process dropped one). The +// read puts back to pending the unacknowledged processing rows whose owner +// has no live row; with legacy also those without an owner, which an older +// version claimed. It returns the file's count, -1 when nothing was +// committed. A table whose requeue fails does not stop the rest; its error +// is returned. // // Gap guard: a gap since the last sweep by any process longer than gapLimit // means no process could beat (a suspended host, a stopped process), and a @@ -150,7 +154,7 @@ func (f *file) sweep(qs []*Queue, legacy bool) (wake bool, err error) { // moves once between two beats of its owner: a dead owner's time runs out // even when every process that opens the file is gone before its second // sweep. With no clock row yet, no owner is taken for dead. -func (f *file) sweepLocked(qs []*Queue, legacy bool) (moved int64, err error) { +func (f *file) sweepLocked(qs []*Queue, legacy, track bool) (moved int64, err error) { tx, err := f.db.Begin() if err != nil { return -1, err @@ -180,6 +184,18 @@ func (f *file) sweepLocked(qs []*Queue, legacy bool) (moved int64, err error) { } } + scan := legacy || track && moved != f.seenMoved + if reap { + res, err := tx.Exec("DELETE FROM __sqliteq_owners WHERE expires_at <= ?", now) + if err != nil { + return -1, err + } + if n, _ := res.RowsAffected(); n > 0 { + scan = true + moved++ // the other processes read their tables too + } + } + // NULL NOT IN an empty set is true, so the ticker sweep names NOT NULL const dead = "owner NOT IN (SELECT id FROM __sqliteq_owners WHERE expires_at > ?)" var where string @@ -193,7 +209,7 @@ func (f *file) sweepLocked(qs []*Queue, legacy bool) (moved int64, err error) { where = "owner IS NULL" } var errs []error - if where != "" { + if where != "" && scan { at, some := time.UnixMilli(now).UTC(), false for _, q := range qs { res, err := tx.Exec(fmt.Sprintf("UPDATE %s SET status = 'pending', owner = NULL, updated_at = ? WHERE status = 'processing' AND ack = 0 AND %s", @@ -214,11 +230,6 @@ func (f *file) sweepLocked(qs []*Queue, legacy bool) (moved int64, err error) { ON CONFLICT (id) DO UPDATE SET swept_at = excluded.swept_at, moved = excluded.moved`, now, moved); err != nil { return -1, err } - if reap { - if _, err := tx.Exec("DELETE FROM __sqliteq_owners WHERE expires_at <= ?", now); err != nil { - return -1, err - } - } if err := tx.Commit(); err != nil { return -1, err } @@ -228,9 +239,9 @@ func (f *file) sweepLocked(qs []*Queue, legacy bool) (moved int64, err error) { return moved, errors.Join(errs...) } -// sweepLoop sweeps every table the process has open on the file, every -// sweepEvery until the last release, and wakes its queues when any process -// requeued rows. It runs in every process with a queue open, also one that +// sweepLoop sweeps the file every sweepEvery until the last release, for +// every table the process has open on it, and wakes its queues when any +// process dropped a dead owner or requeued rows. It runs in every process with a queue open, also one that // never claims: VarMQ pops only when Len is above 0, so a consumer that // waits on an empty queue would never start it, and a dead owner's rows // would stay. It takes no registry lock: an open of another file that waits diff --git a/owner_test.go b/owner_test.go index f4478c6..38ea3a9 100644 --- a/owner_test.go +++ b/owner_test.go @@ -539,29 +539,56 @@ func TestHeartbeatOption(t *testing.T) { } } -// The sweep and the hand-back read the processing index, not the pending -// backlog, and a finish stays a rowid lookup. -func TestOwnerQueryPlans(t *testing.T) { +// A finish stays a rowid lookup with the owner match. +func TestFinishQueryPlan(t *testing.T) { q, qs := openQueue(t) defer qs.Close() - const set = `UPDATE "q" SET status = 'pending', owner = NULL WHERE status = 'processing' AND ack = 0 AND ` - for query, want := range map[string]string{ - set + "(owner IS NULL OR owner NOT IN (SELECT id FROM __sqliteq_owners WHERE expires_at > 0))": "USING INDEX q__sqliteq_processing", - set + "owner IS NOT NULL AND owner NOT IN (SELECT id FROM __sqliteq_owners WHERE expires_at > 0)": "USING INDEX q__sqliteq_processing", - set + "owner = 'x'": "USING INDEX q__sqliteq_processing", - `DELETE FROM "q" WHERE id = 1 AND status = 'processing' AND owner = 'x' RETURNING job_id, data`: "USING INTEGER PRIMARY KEY (rowid=?)", + for _, query := range []string{ + `DELETE FROM "q" WHERE id = 1 AND status = 'processing' AND owner = 'x' RETURNING job_id, data`, + `UPDATE "q" SET status = 'completed', ack = 1 WHERE id = 1 AND status = 'processing' AND owner = 'x' RETURNING job_id, data`, } { var id, parent, unused int var detail string if err := q.f.db.QueryRow("EXPLAIN QUERY PLAN "+query).Scan(&id, &parent, &unused, &detail); err != nil { t.Fatal(err) } - if !strings.Contains(detail, want) { - t.Errorf("%s\n\tplan %q, want %q", query, detail, want) + if !strings.Contains(detail, "USING INTEGER PRIMARY KEY (rowid=?)") { + t.Errorf("%s\n\tplan %q", query, detail) } } } +// A sweep with no dead owner reads no queue table, and a dead owner dropped +// by one process makes every other process read its tables once. +func TestSweepReadsTablesOnDeath(t *testing.T) { + a, qs := openQueue(t) + defer qs.Close() + sweep := func(q *Queue) { + t.Helper() + if _, err := q.f.sweep([]*Queue{q}, false); err != nil { + t.Fatal(err) + } + } + // A row no live owner holds, with no owner row to drop: only a read of + // the table finds it + mustExec(t, a.f.db, `INSERT INTO "q" (data, status, ack, owner) VALUES (x'00', 'processing', 0, 'gone')`) + sweep(a) + if status, _ := rowOf(t, a.f.db); status != "processing" { + t.Fatalf("Expected a sweep with no dead owner to read no table, got %s", status) + } + // Another process drops a dead owner: it has the table open too, so + // take the row out of its reach by the count alone + b := otherProcess(t, a) + mustExec(t, a.f.db, `UPDATE "q" SET status = 'pending'`) + mustExec(t, a.f.db, "INSERT INTO __sqliteq_owners (id, expires_at) VALUES ('dead', 0)") + sweep(b) + mustExec(t, a.f.db, `UPDATE "q" SET status = 'processing', owner = 'gone'`) + sweep(a) + if status, _ := rowOf(t, a.f.db); status != "pending" { + t.Fatalf("Expected the count to make this process read its table, got %s", status) + } +} + // TestHelperProcess is the second process of TestKilledProcess: a VarMQ // worker that takes one job and never ends it. func TestHelperProcess(t *testing.T) { From 8bed901c3309e85b565aa575730b2129281eebb0 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Wed, 11 Nov 2026 21:20:48 +0600 Subject: [PATCH 09/10] fix: list a table before its first sweep, read tables once per timeout Review findings on the index removal: - a tick with a table list from before an open could use up the count and never read the new table; the list now holds the table before its first sweep and the tick reads it under the write mutex - a row claimed just as its owner row was dropped had no later read; the loop now reads every open table once per timeout Refs #27 --- CHANGELOG.md | 2 +- README.md | 4 +-- owner.go | 44 +++++++++++++++++++------------ owner_test.go | 73 +++++++++++++++++++++++++++++++-------------------- registry.go | 18 ++++++++----- 5 files changed, 87 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f57371..7fca9c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,7 +92,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Owner id per process and file; column `owner` on queue tables, set by a claim, NULL on pending rows. (#28) - Tables `__sqliteq_owners` (heartbeats) and `__sqliteq_clock` (last sweep), per database file. (#28) -- No index on processing rows: a sweep scans a queue table only on its first open and when a heartbeat expired, so a claim and an ack maintain nothing new. (#28) +- No index on processing rows: a sweep scans a queue table only on its first open, when a heartbeat expired and once per timeout, so a claim and an ack maintain nothing new. (#28) - Only a process that claims writes a heartbeat. Producers, inspectors and pausers write none. (#28) - Sweep on a queue's first open and every sixth of the timeout, in every process with a queue open: requeues processing items whose owner has no live heartbeat. (#28) - A dead process's items return about one timeout after its last heartbeat, with no reopen. (#28) diff --git a/README.md b/README.md index bab3a8f..127745d 100644 --- a/README.md +++ b/README.md @@ -419,7 +419,7 @@ On top of that table: - Two partial indexes: one covers the pending items in queue order. A unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on. Processing items have no index, so a claim and an ack do not pay for one. - `__sqliteq_owners` holds one heartbeat row per process that claims (`id`, `expires_at`, `shifted`). `__sqliteq_clock` holds when any process last swept and a count of the requeues. Both are per database file. -- One sweep goroutine per open file, in every process with a queue open on it. Each sixth of the heartbeat timeout it renews the heartbeat and deletes expired heartbeats, in one transaction on the two small tables. It reads a queue table, one scan, only when a heartbeat expired: the process that deletes it moves a count in `__sqliteq_clock`, and each process that sees the count move puts the processing items of dead owners in its open queues back to pending. The first open of a queue reads its table too, as recovery always did. +- One sweep goroutine per open file, in every process with a queue open on it. Each sixth of the heartbeat timeout it renews the heartbeat and deletes expired heartbeats, in one transaction on the two small tables. It reads a queue table, one scan, only when a heartbeat expired: the process that deletes it moves a count in `__sqliteq_clock`, and each process that sees the count move puts the processing items of dead owners in its open queues back to pending. The first open of a queue reads its table too, as recovery always did, and each process reads its open tables once per timeout as a safety net. - A claim stores the owner id, and a finish matches it (`AND owner = ?`). The first claim of a process, and a claim after a stall of two sweep intervals, runs a sweep transaction first, which writes the heartbeat. - A claim or a pop is one `UPDATE ... RETURNING` or `DELETE ... RETURNING` statement. It selects nothing while the queue is paused. - `__sqliteq_meta` holds the totals and the pause; a trigger counts each stored add. @@ -439,7 +439,7 @@ On top of that table: - **At least once, not exactly once.** A job runs again when its process stops before the job is acknowledged. Make handlers idempotent. - **No visibility timeout.** A hung worker keeps its job while its process is alive. Items come back only when their process dies or closes the store. - **Recovery waits for the heartbeat.** After a crash, the items of the dead process come back one heartbeat timeout later, also for a single process that restarts at once. See [Processes and files](#processes-and-files). -- **A sweep in every process.** Each process with a queue open writes to each open file every sixth of the heartbeat timeout (5 s by default). When a process dies, each live process scans its open queue tables once, under the write lock. +- **A sweep in every process.** Each process with a queue open writes to each open file every sixth of the heartbeat timeout (5 s by default). Each process also scans its open queue tables under the write lock: once per timeout, and once more when a process dies. - **A stale ack of the same process counts.** A process that froze past the timeout and took the same item again can end the second run with the ack of the first. - **One version per file.** Processes of an older version on the same file still requeue the jobs of live processes. - **Polling, not push.** A queue object with watches or `OnCancel` hooks reads the file every 50 ms, in every process. Outcomes, cancels and a `Resume` from another process arrive within about 50 ms. diff --git a/owner.go b/owner.go index 9bcdf59..194dbb7 100644 --- a/owner.go +++ b/owner.go @@ -24,7 +24,8 @@ import ( // first open, as recovery always did, and when an owner died: the sweep // that drops a dead owner's row moves the count in __sqliteq_clock, and // every process that sees the count move reads its open tables once and -// wakes its queues. +// wakes its queues. Each process also reads them once per timeout, for a +// row whose owner row went without the count moving. // // Times are the host's wall clock in Unix ms. __sqliteq_clock also holds // when any process last swept (see sweepLocked for the gap guard). @@ -103,23 +104,28 @@ func (f *file) beat() error { } first := !f.claimed f.claimed = true - if _, err := f.sweepLocked(nil, false, false); err != nil { + if _, err := f.sweepLocked(nil, false, false, false); err != nil { f.claimed = !first return err } return nil } -// sweep runs sweepLocked under the write mutex. wake reports that some -// process requeued rows in the file since this one last looked: its queues -// may have pending rows no local add announced. -func (f *file) sweep(qs []*Queue, legacy bool) (wake bool, err error) { +// sweep runs sweepLocked under the write mutex; a nil qs is every queue the +// process has open, read under the mutex so no open falls between. full +// reads the tables whatever the count says. wake reports that some process +// dropped an owner or requeued rows in the file since this one last looked: +// its queues may have pending rows no local add announced. +func (f *file) sweep(qs []*Queue, legacy, full bool) (wake bool, err error) { f.mu.Lock() defer f.mu.Unlock() if f.closing { return false, errClosed } - moved, err := f.sweepLocked(qs, legacy, f.seen) + if qs == nil { + qs = *f.swept.Load() + } + moved, err := f.sweepLocked(qs, legacy || full, legacy, f.seen) if moved < 0 { return false, err // nothing committed } @@ -138,10 +144,10 @@ func (f *file) sweep(qs []*Queue, legacy bool) (wake bool, err error) { // sweepLocked is one write transaction: it moves the clock, renews this // process's owner row if it has one and drops the owner rows that ran out. -// It reads the tables of qs, a scan each, only when it has to: on a first -// open (legacy), when it dropped an owner row, or when track is set and the -// file's count moved since f.seenMoved (another process dropped one). The -// read puts back to pending the unacknowledged processing rows whose owner +// It reads the tables of qs, a scan each, only when it has to: when scan is +// set (a first open, the loop's read once per timeout), when it dropped an +// owner row, or when track is set and the file's count moved since +// f.seenMoved (another process dropped one). The read puts back to pending the unacknowledged processing rows whose owner // has no live row; with legacy also those without an owner, which an older // version claimed. It returns the file's count, -1 when nothing was // committed. A table whose requeue fails does not stop the rest; its error @@ -154,7 +160,7 @@ func (f *file) sweep(qs []*Queue, legacy bool) (wake bool, err error) { // moves once between two beats of its owner: a dead owner's time runs out // even when every process that opens the file is gone before its second // sweep. With no clock row yet, no owner is taken for dead. -func (f *file) sweepLocked(qs []*Queue, legacy, track bool) (moved int64, err error) { +func (f *file) sweepLocked(qs []*Queue, scan, legacy, track bool) (moved int64, err error) { tx, err := f.db.Begin() if err != nil { return -1, err @@ -184,7 +190,7 @@ func (f *file) sweepLocked(qs []*Queue, legacy, track bool) (moved int64, err er } } - scan := legacy || track && moved != f.seenMoved + scan = scan || track && moved != f.seenMoved if reap { res, err := tx.Exec("DELETE FROM __sqliteq_owners WHERE expires_at <= ?", now) if err != nil { @@ -250,21 +256,25 @@ func (f *file) sweepLoop() { defer close(f.done) t := time.NewTicker(f.sweepEvery()) defer t.Stop() - for { + for n := 1; ; n++ { select { case <-f.stop: return case <-t.C: } - qs := *f.swept.Load() + // Once per timeout the tables are read whatever the count says: a + // row can carry an owner whose row went without the count moving + // (claimed after a stall, just as the row was dropped). + // ponytail: one scan of each open table per timeout. Upgrade path: + // the claim checks its owner row in the same statement. // A failed sweep (a busy file) is tried again at the next tick - wake, err := f.sweep(qs, false) + wake, err := f.sweep(nil, false, n%6 == 0) if err != nil && !errors.Is(err, errClosed) { log.Printf("sqliteq: sweep of %s: %v", f.key, err) } if wake { - for _, q := range qs { + for _, q := range *f.swept.Load() { q.wake() } } diff --git a/owner_test.go b/owner_test.go index 38ea3a9..4e4ca7a 100644 --- a/owner_test.go +++ b/owner_test.go @@ -25,7 +25,7 @@ func (q *Queue) requeueNoAck() error { if err != nil { return err } - _, err = f.sweep([]*Queue{q}, true) + _, err = f.sweep([]*Queue{q}, true, false) return err } @@ -241,7 +241,7 @@ func TestWakeAfterOthersRequeue(t *testing.T) { defer qs.Close() sweep := func(q *Queue) bool { t.Helper() - wake, err := q.f.sweep([]*Queue{q}, false) + wake, err := q.f.sweep([]*Queue{q}, false, false) if err != nil { t.Fatal(err) } @@ -282,17 +282,44 @@ func TestOpenKeepsWake(t *testing.T) { claim1(t, dead) kill(dead.f) mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = 0") - if _, err := observer.f.sweep([]*Queue{observer}, false); err != nil { + if _, err := observer.f.sweep([]*Queue{observer}, false, false); err != nil { t.Fatal(err) } if _, err := qs.NewQueue("y"); err != nil { t.Fatal(err) } - if wake, err := a.f.sweep([]*Queue{a}, false); err != nil || !wake { + if wake, err := a.f.sweep([]*Queue{a}, false, false); err != nil || !wake { t.Fatalf("Expected the next sweep to wake after an open in between, got %t %v", wake, err) } } +// A tick sees a table from before its first sweep: an owner that dies +// between that sweep and the tick is found in it. +func TestOpenListsTableBeforeSweep(t *testing.T) { + qs := mustNew(t, filepath.Join(t.TempDir(), "one.db"), WithSingleFile()) + defer qs.Close() + a, err := qs.NewQueue("q") + if err != nil { + t.Fatal(err) + } + dead := otherProcess(t, a) + claim1(t, dead) + kill(dead.f) + if _, err := qs.NewQueue("y"); err != nil { // the owner is live still + t.Fatal(err) + } + if n := len(*a.f.swept.Load()); n != 2 { + t.Fatalf("Expected both tables in the loop's list, got %d", n) + } + mustExec(t, a.f.db, "UPDATE __sqliteq_owners SET expires_at = 0") + if _, err := a.f.sweep(nil, false, false); err != nil { + t.Fatal(err) + } + if a.Len() != 1 { + t.Fatal("Expected the tick to requeue the dead owner's row") + } +} + // After a crash the restarted process gets its old rows back one timeout // later, not at open. func TestOwnerRestart(t *testing.T) { @@ -350,7 +377,7 @@ func TestLegacyRows(t *testing.T) { a, qs := openQueue(t) defer qs.Close() mustExec(t, a.f.db, `INSERT INTO "q" (data, status, ack) VALUES (x'00', 'processing', 0)`) - if _, err := a.f.sweep([]*Queue{a}, false); err != nil { + if _, err := a.f.sweep([]*Queue{a}, false, false); err != nil { t.Fatal(err) } if status, _ := rowOf(t, a.f.db); status != "processing" { @@ -410,7 +437,7 @@ func TestClockGap(t *testing.T) { } sweep := func(t *testing.T, a *Queue) (status string, expires int64) { t.Helper() - if _, err := a.f.sweep([]*Queue{a}, false); err != nil { + if _, err := a.f.sweep([]*Queue{a}, false, false); err != nil { t.Fatal(err) } status, _ = rowOf(t, a.f.db) @@ -458,7 +485,7 @@ func TestClockGap(t *testing.T) { t.Run("unreadable clock", func(t *testing.T) { a, _ := setup(t) mustExec(t, a.f.db, "UPDATE __sqliteq_clock SET swept_at = 'x'") - if _, err := a.f.sweep([]*Queue{a}, false); err != nil { + if _, err := a.f.sweep([]*Queue{a}, false, false); err != nil { t.Fatalf("Expected the sweep to repair the clock, got %v", err) } }) @@ -487,7 +514,7 @@ func TestProducerOnly(t *testing.T) { } stats(t, a) a.Dequeue() - if _, err := a.f.sweep([]*Queue{a}, false); err != nil { + if _, err := a.f.sweep([]*Queue{a}, false, false); err != nil { t.Fatal(err) } if n := ownerRows(t, a.f.db); n != 0 || a.f.claimed { @@ -539,25 +566,6 @@ func TestHeartbeatOption(t *testing.T) { } } -// A finish stays a rowid lookup with the owner match. -func TestFinishQueryPlan(t *testing.T) { - q, qs := openQueue(t) - defer qs.Close() - for _, query := range []string{ - `DELETE FROM "q" WHERE id = 1 AND status = 'processing' AND owner = 'x' RETURNING job_id, data`, - `UPDATE "q" SET status = 'completed', ack = 1 WHERE id = 1 AND status = 'processing' AND owner = 'x' RETURNING job_id, data`, - } { - var id, parent, unused int - var detail string - if err := q.f.db.QueryRow("EXPLAIN QUERY PLAN "+query).Scan(&id, &parent, &unused, &detail); err != nil { - t.Fatal(err) - } - if !strings.Contains(detail, "USING INTEGER PRIMARY KEY (rowid=?)") { - t.Errorf("%s\n\tplan %q", query, detail) - } - } -} - // A sweep with no dead owner reads no queue table, and a dead owner dropped // by one process makes every other process read its tables once. func TestSweepReadsTablesOnDeath(t *testing.T) { @@ -565,7 +573,7 @@ func TestSweepReadsTablesOnDeath(t *testing.T) { defer qs.Close() sweep := func(q *Queue) { t.Helper() - if _, err := q.f.sweep([]*Queue{q}, false); err != nil { + if _, err := q.f.sweep([]*Queue{q}, false, false); err != nil { t.Fatal(err) } } @@ -576,6 +584,15 @@ func TestSweepReadsTablesOnDeath(t *testing.T) { if status, _ := rowOf(t, a.f.db); status != "processing" { t.Fatalf("Expected a sweep with no dead owner to read no table, got %s", status) } + // The loop's read once per timeout finds it + if _, err := a.f.sweep(nil, false, true); err != nil { + t.Fatal(err) + } + if status, _ := rowOf(t, a.f.db); status != "pending" { + t.Fatalf("Expected the full read to requeue the row, got %s", status) + } + mustExec(t, a.f.db, `UPDATE "q" SET status = 'processing', owner = 'gone'`) + sweep(a) // Another process drops a dead owner: it has the table open too, so // take the row out of its reach by the count alone b := otherProcess(t, a) diff --git a/registry.go b/registry.go index 2f95e4e..da78f1d 100644 --- a/registry.go +++ b/registry.go @@ -240,8 +240,19 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) // First open: rows of dead owners, and rows an older version left // without an owner, go back to pending + // The loop's list holds the table before its first sweep, so an owner + // that dies between that sweep and the next tick is found by the tick + old := f.swept.Load() + qs := []*Queue{h.queue()} + for k, o := range f.handles { + if k != key { + qs = append(qs, o.queue()) + } + } + f.swept.Store(&qs) if !f.recovered[key] { - if _, err := f.sweep([]*Queue{h.queue()}, true); err != nil { + if _, err := f.sweep([]*Queue{h.queue()}, true, false); err != nil { + f.swept.Store(old) h.queue().Close() // stops a pause poll return nil, fmt.Errorf("failed to requeue unacknowledged rows: %w", err) } @@ -249,11 +260,6 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) } f.handles[key] = h - qs := make([]*Queue, 0, len(f.handles)) - for _, h := range f.handles { - qs = append(qs, h.queue()) - } - f.swept.Store(&qs) if !f.sweeping { f.sweeping = true go f.sweepLoop() From 88c045805f6fe1aa9647b666426ec9d7523ad33d Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Thu, 12 Nov 2026 18:28:34 +0600 Subject: [PATCH 10/10] test: cover the sweep, beat and hand-back error paths --- owner_test.go | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/owner_test.go b/owner_test.go index 4e4ca7a..32d6d1f 100644 --- a/owner_test.go +++ b/owner_test.go @@ -3,11 +3,15 @@ package sqliteq import ( "bufio" "database/sql" + "errors" "fmt" + "log" "os" "os/exec" "path/filepath" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -695,3 +699,207 @@ func TestKilledProcess(t *testing.T) { t.Fatal("the killed owner's job did not run in the survivor") } } + +// brokenClock makes every write of the clock row fail. +const brokenClock = "CREATE TRIGGER nope BEFORE UPDATE ON __sqliteq_clock BEGIN SELECT RAISE(ABORT, 'nope'); END" + +// A sweep that cannot finish a step returns the error and commits nothing; +// a table that fails does not stop the sweep. +func TestSweepErrors(t *testing.T) { + for _, c := range []struct { + name string + claimed bool + commits bool + harm func(t *testing.T, q *Queue) + }{ + {name: "begin", harm: func(t *testing.T, q *Queue) { q.f.db.Close() }}, + {name: "clock read", harm: func(t *testing.T, q *Queue) { mustExec(t, q.f.db, "DROP TABLE __sqliteq_clock") }}, + {name: "gap", harm: func(t *testing.T, q *Queue) { + mustExec(t, q.f.db, "UPDATE __sqliteq_clock SET swept_at = 0") + mustExec(t, q.f.db, "DROP TABLE __sqliteq_owners") + }}, + {name: "beat", claimed: true, harm: func(t *testing.T, q *Queue) { mustExec(t, q.f.db, "DROP TABLE __sqliteq_owners") }}, + {name: "reap", harm: func(t *testing.T, q *Queue) { mustExec(t, q.f.db, "DROP TABLE __sqliteq_owners") }}, + {name: "clock write", harm: func(t *testing.T, q *Queue) { mustExec(t, q.f.db, brokenClock) }}, + {name: "table", commits: true, harm: func(t *testing.T, q *Queue) { mustExec(t, q.f.db, "DROP TABLE "+q.table) }}, + } { + t.Run(c.name, func(t *testing.T) { + q, err := fastStore(t, t.TempDir(), defaultHeartbeat).NewQueue("q") + if err != nil { + t.Fatal(err) + } + if c.claimed { + claim1(t, q) + } + c.harm(t, q) + + q.f.mu.Lock() + moved, err := q.f.sweepLocked([]*Queue{q}, true, false, false) + q.f.mu.Unlock() + if err == nil || (moved >= 0) != c.commits { + t.Fatalf("moved %d, err %v; want an error, committed %t", moved, err, c.commits) + } + if _, err := q.f.sweep(nil, false, true); err == nil { + t.Fatal("sweep returned no error") + } + }) + } +} + +// A claim whose beat fails takes no row and leaves the owner unclaimed; a +// file that is closing takes no claim and no sweep. +func TestBeatErrors(t *testing.T) { + q, err := fastStore(t, t.TempDir(), defaultHeartbeat).NewQueue("q") + if err != nil { + t.Fatal(err) + } + q.enqueue([]byte("job"), 0, "") + f := q.f + + mustExec(t, f.db, brokenClock) + if _, ok, _ := q.DequeueWithAckId(); ok || f.claimed { + t.Fatalf("claimed with a failed beat (claimed %t)", f.claimed) + } + mustExec(t, f.db, "DROP TRIGGER nope") + + f.mu.Lock() + f.closing = true + f.mu.Unlock() + if _, ok, _ := q.DequeueWithAckId(); ok { + t.Fatal("claimed on a closing file") + } + if _, err := f.sweep(nil, false, false); !errors.Is(err, errClosed) { + t.Fatalf("sweep on a closing file: %v", err) + } + f.mu.Lock() + f.closing = false + f.mu.Unlock() + + if _, ok, _ := q.DequeueWithAckId(); !ok { + t.Fatal("no claim after the file recovered") + } +} + +// A hand-back that fails changes nothing: the rows return one timeout later. +func TestHandBackErrors(t *testing.T) { + for _, c := range []struct { + name string + harm string + tables []string + }{ + {name: "table", tables: []string{`"nope"`}}, + {name: "clock", harm: brokenClock}, + {name: "owner row", harm: "DROP TABLE __sqliteq_owners"}, + } { + t.Run(c.name, func(t *testing.T) { + q, err := fastStore(t, t.TempDir(), defaultHeartbeat).NewQueue("q") + if err != nil { + t.Fatal(err) + } + claim1(t, q) + if c.harm != "" { + mustExec(t, q.f.db, c.harm) + } + q.f.handBack(append([]string{q.table}, c.tables...)) + if status, owner := rowOf(t, q.f.db); status != "processing" || owner.String != q.f.owner { + t.Fatalf("row is %s, owner %q; want processing under this owner", status, owner.String) + } + }) + } +} + +// A first open whose sweep fails gives no handle and leaves the loop's list +// as it was; the next open works. +func TestOpenSweepError(t *testing.T) { + qs := fastStore(t, filepath.Join(t.TempDir(), "one.db"), defaultHeartbeat, WithSingleFile()) + q, err := qs.NewQueue("q") + if err != nil { + t.Fatal(err) + } + mustExec(t, q.f.db, brokenClock) + if _, err := qs.NewQueue("r"); err == nil { + t.Fatal("opened a queue whose first sweep failed") + } + if n := len(*q.f.swept.Load()); n != 1 { + t.Fatalf("loop lists %d queues, want 1", n) + } + mustExec(t, q.f.db, "DROP TRIGGER nope") + if _, err := qs.NewQueue("r"); err != nil { + t.Fatal(err) + } + if n := len(*q.f.swept.Load()); n != 2 { + t.Fatalf("loop lists %d queues, want 2", n) + } +} + +// lockedBuf is a log sink the test reads while the loop writes. +type lockedBuf struct { + mu sync.Mutex + b strings.Builder +} + +func (l *lockedBuf) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.Write(p) +} + +func (l *lockedBuf) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.String() +} + +// The loop logs a sweep that fails and tries again at the next tick. +func TestSweepLoopLogsError(t *testing.T) { + var buf lockedBuf + log.SetOutput(&buf) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + + q, err := fastStore(t, t.TempDir(), 60*time.Millisecond).NewQueue("q") + if err != nil { + t.Fatal(err) + } + mustExec(t, q.f.db, brokenClock) + eventually(t, "the failed sweep is logged", func() bool { return strings.Contains(buf.String(), "sqliteq: sweep of") }) + + mustExec(t, q.f.db, "DROP TRIGGER nope") + claim1(t, q) + other := otherProcess(t, q) + kill(q.f) + eventually(t, "the loop sweeps again", func() bool { + status, _ := rowOf(t, other.f.db) + return status == "pending" + }) +} + +// wake calls no hook on a paused or a closed queue. +func TestWakeSkipsPausedAndClosed(t *testing.T) { + q, err := fastStore(t, t.TempDir(), defaultHeartbeat).NewQueue("q") + if err != nil { + t.Fatal(err) + } + var woke atomic.Int32 + q.OnPauseChange(func(paused bool) { + if !paused { + woke.Add(1) + } + }) + q.wake() + if woke.Load() != 1 { + t.Fatalf("woke %d times, want 1", woke.Load()) + } + if err := q.Pause(); err != nil { + t.Fatal(err) + } + q.wake() + if err := q.Resume(); err != nil { + t.Fatal(err) + } + n := woke.Load() // Resume may call the hook itself + q.Close() + q.wake() + if woke.Load() != n || n > 2 { + t.Fatalf("woke %d times after pause and close, was %d", woke.Load(), n) + } +}