Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Requires Go 1.27 (was 1.24): `go.mod` requires VarMQ, which needs it. (#22)
- Queue name must not contain `__sqliteq_`, in both layouts. (#22)
- Reason: a queue's indexes, side tables and triggers use that part in their names. (#22)
- Recovery is by owner and heartbeat, not on open. After a crash, unacknowledged items return one heartbeat timeout (default 30s) after the dead process's last heartbeat, also for a single process that restarts at once. (#28)
- Mixed versions on one file are unsafe: an older process still requeues every processing item on open, and its items carry no owner. Upgrade all processes of a file together. (#28)
- `OnPauseChange` hooks also get `false` when a sweep puts items back into an unpaused queue. (#28)

### Added

Expand Down Expand Up @@ -87,12 +90,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Pause outlives restarts and `Purge`. (#23)
- Refused pop sets `Paused`; while set, the queue object polls the row every 50 ms for a remote `Resume`. (#23)

- Owner id per process and file; column `owner` on queue tables, set by a claim, NULL on pending rows. (#28)
- Tables `__sqliteq_owners` (heartbeats) and `__sqliteq_clock` (last sweep), per database file. (#28)
- No index on processing rows: a sweep scans a queue table only on its first open, when a heartbeat expired and once per timeout, so a claim and an ack maintain nothing new. (#28)
- Only a process that claims writes a heartbeat. Producers, inspectors and pausers write none. (#28)
- Sweep on a queue's first open and every sixth of the timeout, in every process with a queue open: requeues processing items whose owner has no live heartbeat. (#28)
- A dead process's items return about one timeout after its last heartbeat, with no reopen. (#28)
- Requeue count in `__sqliteq_clock`: each process wakes its bound VarMQ workers at its next sweep, whichever process requeued. (#28)
- Last close of a file in the process (last `Queues`, and the `WorkflowStore` under `WithSingleFile`) requeues its own unacknowledged items at once and deletes its heartbeat. (#28)
- Clock-gap guard: a gap between sweeps above a third of the timeout, or a clock step back, moves every heartbeat by the gap, once per heartbeat until its owner renews it. (#28)
- `WithHeartbeatTimeout(d)` store option: default 30s, at least 5s. Must match for every `New` in the process on one file. (#28)

### Changed

- `Acknowledge`, `Reject` and every other finish match the owner: one for an item another process took since returns false. (#28)
- Known limit: a process that froze past the timeout and claimed the same item again can end the second run with the first run's ack. No attempt counter. (#28)
- Transactions begin `IMMEDIATE`. (#22)
- Finish (`Acknowledge`, `Reject`) always runs in a transaction. (#22)
- Pop runs in a transaction while any cancel mark is live. (#22)

### Fixed

- A second process that opened a queue requeued the jobs another live process was running, so they ran twice. (#27, #28)

## [0.3.0] - 2026-10-23

### ⚠️ Breaking Changes
Expand Down
44 changes: 35 additions & 9 deletions README.md

Large diffs are not rendered by default.

67 changes: 67 additions & 0 deletions bench_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package sqliteq

import (
"fmt"
"testing"
)

// benchQueue opens a queue with n pending rows, filled in one statement.
func benchQueue(b *testing.B, n int) *Queue {
b.Helper()
qs, err := New(b.TempDir())
if err != nil {
b.Fatal(err)
}
b.Cleanup(func() { qs.Close() })
q, err := qs.NewQueue("bench")
if err != nil {
b.Fatal(err)
}
if _, err := q.f.db.Exec(fmt.Sprintf(`INSERT INTO %s (data, status, ack, created_at, updated_at)
WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < ?)
SELECT x'6a6f62', 'pending', 0, 0, 0 FROM c`, q.table), n); err != nil {
b.Fatal(err)
}
return q
}

// backlog is the pending rows left behind the ones a benchmark takes
const backlog = 100_000

func BenchmarkClaim(b *testing.B) {
q := benchQueue(b, b.N+backlog)
b.ResetTimer()
for range b.N {
if _, ok, _ := q.DequeueWithAckId(); !ok {
b.Fatal("empty")
}
}
}

func BenchmarkAck(b *testing.B) {
q := benchQueue(b, b.N+backlog)
items, acks := make([]any, b.N), make([]string, b.N)
if n, _, err := q.DequeueBatchWithAckIds(items, acks); err != nil || n != b.N {
b.Fatal(n, err)
}
b.ResetTimer()
for _, id := range acks {
if !q.Acknowledge(id) {
b.Fatal("not acknowledged")
}
}
}

// BenchmarkBatchPop pops in batches of 64; one op is one item.
func BenchmarkBatchPop(b *testing.B) {
q := benchQueue(b, b.N+backlog)
items, acks := make([]any, 64), make([]string, 64)
b.ResetTimer()
for left := b.N; left > 0; {
n, _, err := q.DequeueBatchWithAckIds(items[:min(64, left)], acks)
if err != nil || n == 0 {
b.Fatal(n, err)
}
left -= n
}
}
5 changes: 4 additions & 1 deletion migration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,10 @@ func TestMigrateV02File(t *testing.T) {
if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) {
t.Errorf("Round %d: indexes = %v, want %v", round, got, want)
}
for _, side := range []string{name + jobsSuffix, name + cancelledSuffix, name + readdSuffix} {
if ok, err := hasColumn(db, name, "owner"); err != nil || !ok {
t.Errorf("Round %d: expected the owner column on the old table, got %t %v", round, ok, err)
}
for _, side := range []string{name + jobsSuffix, name + cancelledSuffix, name + readdSuffix, "__sqliteq_owners", "__sqliteq_clock"} {
var n int
if err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE name = ?", side).Scan(&n); err != nil || n != 1 {
t.Errorf("Round %d: expected %s on the old file, got %d %v", round, side, n, err)
Expand Down
25 changes: 23 additions & 2 deletions option.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package sqliteq

import "errors"
import (
"errors"
"time"
)

// errNilOption is returned by New, NewQueue and NewPriorityQueue for a nil
// option, which they would otherwise call
Expand All @@ -21,10 +24,11 @@ func WithRemoveOnComplete(remove bool) Option {
type storeConfig struct {
singleFile bool
checkpointPages int
heartbeat time.Duration
}

func defaultStoreConfig() storeConfig {
return storeConfig{checkpointPages: 1000}
return storeConfig{checkpointPages: 1000, heartbeat: defaultHeartbeat}
}

// StoreOption is a function type that can be used to configure a store passed to New
Expand All @@ -44,3 +48,20 @@ func WithCheckpointPages(n int) StoreOption {
c.checkpointPages = n
}
}

// WithHeartbeatTimeout sets how long this process may go without renewing
// its heartbeat in a database file before other processes on the file take
// it for dead and requeue the items it dequeued and has not acknowledged
// (default 30s, at least 5s; New rejects anything lower). The heartbeat is
// renewed every sixth of the timeout, so it runs out only when the process
// dies or freezes. It does not limit how long a job may run.
//
// A renewal waits up to 5s (the busy timeout) behind another process's
// write, so keep the timeout well above that on a file other processes
// write to. A job still running when its process's heartbeat runs out runs
// again elsewhere. Processes with different timeouts can share a file.
func WithHeartbeatTimeout(d time.Duration) StoreOption {
return func(c *storeConfig) {
c.heartbeat = d
}
}
Loading
Loading