From d5494f99fbdacc7050cfe69aa823fda6b3a07404 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 12:46:13 +0600 Subject: [PATCH 1/4] perf: group-commit acks and enqueues per file Concurrent Enqueue and Acknowledge calls on one file now share a transaction. The first caller with no round running leads: it commits what has queued up, at most its own round plus one more, under the file's write mutex, then hands the lead to the oldest waiting caller. Each caller still gets its own result. If the grouped transaction fails, the round rolls back and each call runs in a transaction of its own, so one bad call fails alone. Claims, pops and purge keep taking the write mutex directly. --- commit.go | 111 +++++++++++++++++ commit_test.go | 300 ++++++++++++++++++++++++++++++++++++++++++++++ priority_queue.go | 2 +- queue.go | 63 +++++----- queue_test.go | 15 +++ registry.go | 7 +- 6 files changed, 465 insertions(+), 33 deletions(-) create mode 100644 commit.go create mode 100644 commit_test.go diff --git a/commit.go b/commit.go new file mode 100644 index 0000000..755675c --- /dev/null +++ b/commit.go @@ -0,0 +1,111 @@ +package sqliteq + +import ( + "database/sql" + "sync" +) + +// op is one caller's write waiting for a round of the group commit +type op struct { + run func(tx *sql.Tx) (bool, error) // one caller's statement(s) + done chan result // buffered 1 +} + +// result is what an op's caller receives: its own outcome, or with lead set, +// the turn to commit for the others +type result struct{ ok, lead bool } + +// batcher groups concurrent writes on one file into shared transactions. The +// first caller with no round running leads: it commits what has queued up, at +// most its own round plus one more, then hands the lead to the oldest waiting +// caller, so no caller commits for others for long under steady load. +type batcher struct { + mu sync.Mutex + pending []*op + leading bool + commit func(ops []*op) // sends exactly one {ok} result to every op +} + +// do runs one write in the next round and returns its own outcome +func (b *batcher) do(run func(*sql.Tx) (bool, error)) bool { + o := &op{run: run, done: make(chan result, 1)} + + b.mu.Lock() + b.pending = append(b.pending, o) + if b.leading { + b.mu.Unlock() + r := <-o.done + if !r.lead { + return r.ok + } + // Handed the lead: o is still pending, first in the next round + b.mu.Lock() + } else { + b.leading = true + } + b.lead() + + return (<-o.done).ok +} + +// lead commits up to two rounds, then hands off to the oldest waiting op or +// ends the lead. b.mu is held on entry and released on return; it is never +// held across a commit. +func (b *batcher) lead() { + for range 2 { + round := b.pending + if len(round) == 0 { + break + } + b.pending = nil + b.mu.Unlock() + b.commit(round) + b.mu.Lock() + } + + if len(b.pending) > 0 { + b.pending[0].done <- result{lead: true} + } else { + b.leading = false + } + b.mu.Unlock() +} + +// commitRound runs a round's ops in one transaction under the write mutex. +// If any step fails, nothing of it is kept and each op runs again in a +// transaction of its own, so one bad call fails alone. +func (f *file) commitRound(ops []*op) { + f.mu.Lock() + defer f.mu.Unlock() + f.rounds.Add(1) + + oks := make([]bool, len(ops)) + err := f.runTx(ops, oks) + if err == nil || len(ops) == 1 { + for i, o := range ops { + o.done <- result{ok: err == nil && oks[i]} + } + return + } + + for i, o := range ops { + err := f.runTx(ops[i:i+1], oks[i:i+1]) + o.done <- result{ok: err == nil && oks[i]} + } +} + +// runTx runs ops in one transaction, storing each op's outcome in oks; on an +// error the transaction is rolled back +func (f *file) runTx(ops []*op, oks []bool) error { + tx, err := f.db.Begin() + if err != nil { + return err + } + for i, o := range ops { + if oks[i], err = o.run(tx); err != nil { + tx.Rollback() + return err + } + } + return tx.Commit() +} diff --git a/commit_test.go b/commit_test.go new file mode 100644 index 0000000..dea048c --- /dev/null +++ b/commit_test.go @@ -0,0 +1,300 @@ +package sqliteq + +import ( + "bytes" + "database/sql" + "errors" + "runtime" + "strconv" + "sync" + "testing" + "time" +) + +// goid returns the calling goroutine's id, parsed from its stack header. +func goid() uint64 { + buf := make([]byte, 64) + buf = buf[:runtime.Stack(buf, false)] + buf = bytes.TrimPrefix(buf, []byte("goroutine ")) + id, _ := strconv.ParseUint(string(buf[:bytes.IndexByte(buf, ' ')]), 10, 64) + return id +} + +// waitAll fails the test if wg is not done within d. +func waitAll(t *testing.T, wg *sync.WaitGroup, d time.Duration) { + t.Helper() + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(d): + t.Fatalf("Calls did not return within %v", d) + } +} + +// runOps is a fake commit: it runs every op without a transaction and sends +// each its own result. +func runOps(ops []*op) { + for _, o := range ops { + ok, err := o.run(nil) + o.done <- result{ok: ok && err == nil} + } +} + +func TestBatcherEveryCallGetsItsResult(t *testing.T) { + b := &batcher{commit: runOps} + + const goroutines, calls = 50, 200 + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func(g int) { + defer wg.Done() + for i := 0; i < calls; i++ { + want := i%7 != 0 + if got := b.do(func(*sql.Tx) (bool, error) { return want, nil }); got != want { + t.Errorf("goroutine %d call %d: got %t, want %t", g, i, got, want) + } + } + }(g) + } + waitAll(t, &wg, time.Minute) +} + +func TestBatcherLeaderHandsOff(t *testing.T) { + // Only the leader runs commit, so these need no lock of their own + type round struct { + leader uint64 + callers map[uint64]bool // goroutines with an op in the round + } + var ( + rounds []round + caller map[uint64]bool // the round being committed + ) + b := &batcher{} + b.commit = func(ops []*op) { + caller = map[uint64]bool{} + runOps(ops) // each run records its caller + rounds = append(rounds, round{goid(), caller}) + time.Sleep(50 * time.Microsecond) + } + + const goroutines, calls = 32, 300 + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func() { + defer wg.Done() + me := goid() + for i := 0; i < calls; i++ { + if !b.do(func(*sql.Tx) (bool, error) { caller[me] = true; return true, nil }) { + t.Errorf("call %d failed", i) + } + } + }() + } + waitAll(t, &wg, time.Minute) + + // A goroutine has one call in flight, so a round holding the leader's own + // op starts its turn; a round without it must be the second of that turn + n := 0 + for i, r := range rounds { + n += len(r.callers) + if r.callers[r.leader] { + continue + } + if i == 0 || rounds[i-1].leader != r.leader || !rounds[i-1].callers[r.leader] { + t.Fatalf("round %d: goroutine %d led more than its own round plus one", i, r.leader) + } + } + if n != goroutines*calls { + t.Errorf("Rounds held %d calls, want %d", n, goroutines*calls) + } +} + +// oneRound runs first and, once it leads a round blocked on the file mutex, +// starts rest, so that every call of rest commits in the one round after +// first's. It returns each call's result, first's at index 0. +func oneRound(t *testing.T, f *file, first func() bool, rest []func() bool) []bool { + t.Helper() + waitFor := func(what string, cond func() bool) { + t.Helper() + for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(time.Millisecond) { + f.b.mu.Lock() + ok := cond() + f.b.mu.Unlock() + if ok { + return + } + if time.Now().After(deadline) { + f.mu.Unlock() + t.Fatalf("Timed out waiting for %s", what) + } + } + } + + got := make([]bool, 1+len(rest)) + var wg sync.WaitGroup + call := func(i int, fn func() bool) { + wg.Add(1) + go func() { + defer wg.Done() + got[i] = fn() + }() + } + + before := f.rounds.Load() + f.mu.Lock() + call(0, first) + waitFor("a leader", func() bool { return f.b.leading && len(f.b.pending) == 0 }) + for i, fn := range rest { + call(i+1, fn) + } + waitFor("the calls to queue up", func() bool { return len(f.b.pending) == len(rest) }) + f.mu.Unlock() + waitAll(t, &wg, 10*time.Second) + + if n := f.rounds.Load() - before; n != 2 { + t.Errorf("Expected 2 rounds, got %d", n) + } + return got +} + +// TestGroupCommitOneBadCall checks that an add that fails inside a shared +// round fails alone. +func TestGroupCommitOneBadCall(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + add := func() bool { return q.Enqueue([]byte("x")) } + rest := []func() bool{func() bool { return q.Enqueue(nil) }} + for i := 0; i < 63; i++ { + rest = append(rest, add) + } + + got := oneRound(t, q.f, add, rest) + if got[1] { + t.Error("Enqueue(nil) succeeded") + } + for i, ok := range got { + if i != 1 && !ok { + t.Errorf("Enqueue %d failed", i) + } + } + if n := q.Len(); n != 64 { + t.Errorf("Expected 64 pending items, got %d", n) + } +} + +// TestGroupAckAfterFailedRound shares one round between 32 good acks, 4 acks +// of ids no row has, and a failing add, so the round fails and falls back. +func TestGroupAckAfterFailedRound(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + enqueueN(t, q, 32) + + items, ackIDs := make([]any, 32), make([]string, 32) + if n, _, err := q.DequeueBatchWithAckIds(items, ackIDs); err != nil || n != 32 { + t.Fatalf("Claimed %d, %v; want 32", n, err) + } + + ack := func(id string) func() bool { return func() bool { return q.Acknowledge(id) } } + rest := []func() bool{} + for _, id := range ackIDs[1:] { + rest = append(rest, ack(id)) + } + for i := 0; i < 4; i++ { + rest = append(rest, ack(strconv.Itoa(1_000_000+i))) // numeric, so they reach the round + } + rest = append(rest, func() bool { return q.Enqueue(nil) }) + + got := oneRound(t, q.f, ack(ackIDs[0]), rest) + for i := 0; i < 32; i++ { + if !got[i] { + t.Errorf("Acknowledge(%s) failed", ackIDs[i]) + } + } + for i := 32; i < len(got); i++ { + if got[i] { + t.Errorf("Call %d succeeded, want false", i) + } + } + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Errorf("Expected no rows left, got %d", n) + } +} + +// TestCommitRoundFallsBackPerOp runs a round whose second op errors: the +// grouped transaction rolls back, and each op then commits alone. +func TestCommitRoundFallsBackPerOp(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + add := func(tx *sql.Tx) (bool, error) { + _, err := tx.Stmt(q.insert).Exec([]byte("x"), time.Now(), time.Now()) + return err == nil, err + } + bad := func(*sql.Tx) (bool, error) { return false, errors.New("bad op") } + ops := []*op{ + {run: add, done: make(chan result, 1)}, + {run: bad, done: make(chan result, 1)}, + {run: add, done: make(chan result, 1)}, + } + q.f.commitRound(ops) + + for i, want := range []bool{true, false, true} { + select { + case r := <-ops[i].done: + if r.ok != want || r.lead { + t.Errorf("op %d: got %+v, want ok=%t", i, r, want) + } + default: + t.Errorf("op %d got no result", i) + } + if len(ops[i].done) != 0 { + t.Errorf("op %d got more than one result", i) + } + } + if n := q.Len(); n != 2 { + t.Errorf("Expected 2 pending items, got %d", n) + } +} + +// TestGroupCommitBatches checks that concurrent writers share rounds. +func TestGroupCommitBatches(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + before := q.f.rounds.Load() + ops := concurrentWriters(t, q) + rounds := q.f.rounds.Load() - before + t.Logf("%d writes in %d rounds", ops, rounds) + if rounds >= int64(ops) { + t.Errorf("Expected fewer rounds than %d writes, got %d", ops, rounds) + } +} + +// TestGroupCommitAfterClose closes the store under running writers: every +// call still returns. +func TestGroupCommitAfterClose(t *testing.T) { + q, qs := openQueue(t) + + var wg sync.WaitGroup + for g := 0; g < 16; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for q.Enqueue([]byte("x")) { + q.Acknowledge("1") + } + }() + } + time.Sleep(20 * time.Millisecond) + if err := qs.Close(); err != nil { + t.Errorf("Close failed: %v", err) + } + waitAll(t, &wg, 10*time.Second) +} diff --git a/priority_queue.go b/priority_queue.go index 734bab3..bc2ff32 100644 --- a/priority_queue.go +++ b/priority_queue.go @@ -20,5 +20,5 @@ func newPriorityQueue(f *file, name string, opts ...Option) (*PriorityQueue, err // Lower priority numbers will be dequeued first (0 is highest priority) // Returns true if the operation was successful func (pq *PriorityQueue) Enqueue(item any, priority int) bool { - return pq.enqueue(item, priority) + return pq.enqueue(item, priority, "") } diff --git a/queue.go b/queue.go index 90ac89c..83cd3e7 100644 --- a/queue.go +++ b/queue.go @@ -262,26 +262,26 @@ func parseAckID(s string) (int64, bool) { // Enqueue adds an item to the queue // Returns true if the operation was successful func (q *Queue) Enqueue(item any) bool { - return q.enqueue(item, 0) + return q.enqueue(item, 0, "") } -// enqueue inserts one pending row; priority is stored on priority queues only -func (q *Queue) enqueue(item any, priority int) bool { +// enqueue inserts one pending row in the file's next group commit round; +// priority is stored on priority queues only. id is unused for now. +func (q *Queue) enqueue(item any, priority int, id string) bool { if q.closed.Load() { return false } now := time.Now().UTC() - q.f.mu.Lock() - defer q.f.mu.Unlock() - - var err error - if q.priority { - _, err = q.insert.Exec(item, now, now, priority) - } else { - _, err = q.insert.Exec(item, now, now) - } - return err == nil + return q.f.b.do(func(tx *sql.Tx) (bool, error) { + var err error + if q.priority { + _, err = tx.Stmt(q.insert).Exec(item, now, now, priority) + } else { + _, err = tx.Stmt(q.insert).Exec(item, now, now) + } + return err == nil, err + }) } // claimN moves up to n pending rows to processing in one statement and @@ -406,7 +406,7 @@ func (q *Queue) DequeueBatchWithAckIds(items []any, ackIDs []string) (n, remaini } // Acknowledge marks a processing item as completed, deleting it unless -// WithRemoveOnComplete(false) is set +// WithRemoveOnComplete(false) is set, in the file's next group commit round // Returns true only if exactly one processing row had this ack ID func (q *Queue) Acknowledge(ackID string) bool { id, ok := parseAckID(ackID) @@ -414,24 +414,25 @@ func (q *Queue) Acknowledge(ackID string) bool { return false } - q.f.mu.Lock() - defer q.f.mu.Unlock() - - var ( - result sql.Result - err error - ) - if q.removeOnComplete { - result, err = q.ack.Exec(id) - } else { - result, err = q.ack.Exec(time.Now().UTC(), id) - } - if err != nil { - return false - } + now := time.Now().UTC() + return q.f.b.do(func(tx *sql.Tx) (bool, error) { + var ( + result sql.Result + err error + ) + if q.removeOnComplete { + result, err = tx.Stmt(q.ack).Exec(id) + } else { + result, err = tx.Stmt(q.ack).Exec(now, id) + } + if err != nil { + return false, err + } - n, err := result.RowsAffected() - return err == nil && n == 1 + // No row is an outcome, not an error: it must not fail the round + n, err := result.RowsAffected() + return err == nil && n == 1, err + }) } // Len returns the number of pending items in the queue diff --git a/queue_test.go b/queue_test.go index fb081e8..a340992 100644 --- a/queue_test.go +++ b/queue_test.go @@ -741,6 +741,14 @@ func TestConcurrentWritersOneFile(t *testing.T) { q, qs := openQueue(t) defer qs.Close() + concurrentWriters(t, q) +} + +// concurrentWriters runs 8 producers and 16 ackers against one batch popper +// and checks every job is delivered once and acked. It returns the number of +// writes that go through the group commit: every Enqueue and Acknowledge. +func concurrentWriters(t *testing.T, q *Queue) int { + t.Helper() const producers, perProducer, ackers, batch = 8, 500, 16, 16 const total = producers * perProducer @@ -768,7 +776,13 @@ func TestConcurrentWritersOneFile(t *testing.T) { defer close(drained) defer close(jobs) items, ackIDs := make([]any, batch), make([]string, batch) + // A failed add leaves fewer than total jobs, so the drain is bounded + deadline := time.Now().Add(time.Minute) for got := 0; got < total; { + if time.Now().After(deadline) { + t.Errorf("Drained %d of %d jobs before the deadline", got, total) + return + } n, _, err := q.DequeueBatchWithAckIds(items, ackIDs) if err != nil { t.Errorf("Batch pop failed: %v", err) @@ -812,6 +826,7 @@ func TestConcurrentWritersOneFile(t *testing.T) { if n := countRows(t, q.f.db, "1"); n != 0 { t.Errorf("Expected no rows left, got %d", n) } + return 2 * total } // TestWritesTakeFileMutex holds the file's write mutex and checks that every diff --git a/registry.go b/registry.go index 355a8bc..74d6151 100644 --- a/registry.go +++ b/registry.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "github.com/mattn/go-sqlite3" ) @@ -18,6 +19,8 @@ type file struct { pages int // checkpoint pages it was opened with refs int // Queues values holding it; under registry.mu mu sync.Mutex // write mutex: every write on the file holds it, reads do not + b *batcher // group commit for enqueues and acks + rounds atomic.Int64 // rounds committed by b; read by tests handles map[string]*handle // lower-cased queue name -> handle; under registry.mu recovered map[string]bool // lower-cased table -> recovery done; under registry.mu } @@ -64,7 +67,8 @@ func dsn(path string) string { } // registry holds every open file in the process, keyed by resolved path. -// Lock order is queues.mu, then registry.mu, then file.mu. +// Lock order is queues.mu, then registry.mu, then file.mu. A file's +// batcher.mu is never held across a commit, so never together with file.mu. var registry = struct { mu sync.Mutex files map[string]*file @@ -101,6 +105,7 @@ func acquire(path string, pages int) (*file, error) { handles: map[string]*handle{}, recovered: map[string]bool{}, } + f.b = &batcher{commit: f.commitRound} registry.files[path] = f return f, nil } From ee264a5a72d319506960140c975135afad432f81 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 13:45:32 +0600 Subject: [PATCH 2/4] perf: run a one-op group commit round without a transaction A database/sql Tx costs a cancelable context and a goroutine per transaction. A round of one op has nothing to share, and in the folder layout about half the rounds hold one op, so at GOMAXPROCS=24 the Tx setup cost more than grouping saved (Mixed10Q/Folder-ckpt1000 fell about 19% below the per-call commits). A lone op now runs as one autocommit statement, still under the write mutex. --- commit.go | 14 +++++++++++--- commit_test.go | 27 +++++++++++++++++++++++++++ queue.go | 16 ++++++++++++---- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/commit.go b/commit.go index 755675c..cab4c2d 100644 --- a/commit.go +++ b/commit.go @@ -7,7 +7,7 @@ import ( // op is one caller's write waiting for a round of the group commit type op struct { - run func(tx *sql.Tx) (bool, error) // one caller's statement(s) + run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit done chan result // buffered 1 } @@ -73,15 +73,23 @@ func (b *batcher) lead() { // commitRound runs a round's ops in one transaction under the write mutex. // If any step fails, nothing of it is kept and each op runs again in a -// transaction of its own, so one bad call fails alone. +// transaction of its own, so one bad call fails alone. A round of one op runs +// with a nil tx, as one autocommit statement: a database/sql Tx costs a +// context and a goroutine, which a round with nothing to share does not repay. func (f *file) commitRound(ops []*op) { f.mu.Lock() defer f.mu.Unlock() f.rounds.Add(1) + if len(ops) == 1 { + ok, err := ops[0].run(nil) + ops[0].done <- result{ok: err == nil && ok} + return + } + oks := make([]bool, len(ops)) err := f.runTx(ops, oks) - if err == nil || len(ops) == 1 { + if err == nil { for i, o := range ops { o.done <- result{ok: err == nil && oks[i]} } diff --git a/commit_test.go b/commit_test.go index dea048c..fa4fe91 100644 --- a/commit_test.go +++ b/commit_test.go @@ -263,6 +263,33 @@ func TestCommitRoundFallsBackPerOp(t *testing.T) { } } +// TestCommitRoundOneOpAutocommits checks that a round of one op runs without +// a transaction, and a round of more shares one. +func TestCommitRoundOneOpAutocommits(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + var txs []*sql.Tx + record := func(tx *sql.Tx) (bool, error) { + txs = append(txs, tx) + return true, nil + } + newOp := func() *op { return &op{run: record, done: make(chan result, 1)} } + + q.f.commitRound([]*op{newOp()}) + q.f.commitRound([]*op{newOp(), newOp()}) + + if len(txs) != 3 { + t.Fatalf("Expected 3 runs, got %d", len(txs)) + } + if txs[0] != nil { + t.Error("A round of one op ran in a transaction") + } + if txs[1] == nil || txs[1] != txs[2] { + t.Error("A round of two ops did not share one transaction") + } +} + // TestGroupCommitBatches checks that concurrent writers share rounds. func TestGroupCommitBatches(t *testing.T) { q, qs := openQueue(t) diff --git a/queue.go b/queue.go index 83cd3e7..b2a17c8 100644 --- a/queue.go +++ b/queue.go @@ -274,11 +274,15 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { now := time.Now().UTC() return q.f.b.do(func(tx *sql.Tx) (bool, error) { + insert := q.insert + if tx != nil { + insert = tx.Stmt(insert) + } var err error if q.priority { - _, err = tx.Stmt(q.insert).Exec(item, now, now, priority) + _, err = insert.Exec(item, now, now, priority) } else { - _, err = tx.Stmt(q.insert).Exec(item, now, now) + _, err = insert.Exec(item, now, now) } return err == nil, err }) @@ -420,10 +424,14 @@ func (q *Queue) Acknowledge(ackID string) bool { result sql.Result err error ) + ack := q.ack + if tx != nil { + ack = tx.Stmt(ack) + } if q.removeOnComplete { - result, err = tx.Stmt(q.ack).Exec(id) + result, err = ack.Exec(id) } else { - result, err = tx.Stmt(q.ack).Exec(now, id) + result, err = ack.Exec(now, id) } if err != nil { return false, err From 2db63b7a225189eda176ba81879f1f212eae0bcc Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 13:47:58 +0600 Subject: [PATCH 3/4] perf: let a running caller take the group commit lead A leader ending its turn used to keep the lead set and pass it to the oldest waiting caller, so the next round waited for that goroutine to be woken and scheduled. Now the lead is given up and the oldest waiter is woken for it; a caller that arrives first takes it instead and commits the waiter's op in its own round. A leader still commits at most its own round plus one more. An op's done channel holds two: at most one wake-up and the outcome, so a commit never blocks on a caller that has not woken yet. --- commit.go | 48 ++++++++++++++++++++++++++------------- commit_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 15 deletions(-) diff --git a/commit.go b/commit.go index cab4c2d..429af8b 100644 --- a/commit.go +++ b/commit.go @@ -8,7 +8,7 @@ import ( // op is one caller's write waiting for a round of the group commit type op struct { run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit - done chan result // buffered 1 + done chan result // buffered 2: one lead wake-up and the outcome } // result is what an op's caller receives: its own outcome, or with lead set, @@ -17,8 +17,10 @@ type result struct{ ok, lead bool } // batcher groups concurrent writes on one file into shared transactions. The // first caller with no round running leads: it commits what has queued up, at -// most its own round plus one more, then hands the lead to the oldest waiting -// caller, so no caller commits for others for long under steady load. +// most its own round plus one more, then gives up the lead, so no caller +// commits for others for long under steady load. Whoever comes first takes +// the lead next: a new caller, or the oldest waiting one, which is woken for +// it. The next round never waits on that wake-up while a caller is running. type batcher struct { mu sync.Mutex pending []*op @@ -28,29 +30,45 @@ type batcher struct { // do runs one write in the next round and returns its own outcome func (b *batcher) do(run func(*sql.Tx) (bool, error)) bool { - o := &op{run: run, done: make(chan result, 1)} + o := &op{run: run, done: make(chan result, 2)} b.mu.Lock() b.pending = append(b.pending, o) - if b.leading { + for b.leading { b.mu.Unlock() r := <-o.done if !r.lead { return r.ok } - // Handed the lead: o is still pending, first in the next round + // Woken for the lead: take it if nobody has and o is still first, + // else o is in someone's round and its outcome follows b.mu.Lock() - } else { - b.leading = true + if !b.leading && len(b.pending) > 0 && b.pending[0] == o { + break + } + if !b.leading { + b.mu.Unlock() + return o.outcome() + } } + b.leading = true b.lead() - return (<-o.done).ok + return o.outcome() +} + +// outcome waits for the op's result, skipping a wake-up that came too late +func (o *op) outcome() bool { + for { + if r := <-o.done; !r.lead { + return r.ok + } + } } -// lead commits up to two rounds, then hands off to the oldest waiting op or -// ends the lead. b.mu is held on entry and released on return; it is never -// held across a commit. +// lead commits up to two rounds, then gives up the lead and wakes the oldest +// waiting op, if any, to take it. b.mu is held on entry and released on +// return; it is never held across a commit. func (b *batcher) lead() { for range 2 { round := b.pending @@ -63,10 +81,10 @@ func (b *batcher) lead() { b.mu.Lock() } - if len(b.pending) > 0 { + b.leading = false + // A pending op holds no outcome, so an empty buffer means no wake-up is unread + if len(b.pending) > 0 && len(b.pending[0].done) == 0 { b.pending[0].done <- result{lead: true} - } else { - b.leading = false } b.mu.Unlock() } diff --git a/commit_test.go b/commit_test.go index fa4fe91..87a2f72 100644 --- a/commit_test.go +++ b/commit_test.go @@ -115,6 +115,67 @@ func TestBatcherLeaderHandsOff(t *testing.T) { } } +// TestBatcherHandOffDoesNotWait checks that once a leader has ended its turn, +// the next round does not wait for the waiting op it woke: a caller arriving +// first takes the lead and commits that op with its own. +func TestBatcherHandOffDoesNotWait(t *testing.T) { + gate := make(chan struct{}) + rounds := 0 // touched by the leader only + b := &batcher{} + b.commit = func(ops []*op) { + if rounds < 2 { + <-gate + } + rounds++ + runOps(ops) + } + waitFor := func(what string, cond func() bool) { + t.Helper() + for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(time.Millisecond) { + b.mu.Lock() + ok := cond() + b.mu.Unlock() + if ok { + return + } + if time.Now().After(deadline) { + t.Fatalf("Timed out waiting for %s", what) + } + } + } + ok := func(*sql.Tx) (bool, error) { return true, nil } + + go b.do(ok) // leads round 1, held at the gate + waitFor("round 1", func() bool { return b.leading && len(b.pending) == 0 }) + go b.do(ok) // waits, then commits in round 2 + waitFor("a waiter", func() bool { return len(b.pending) == 1 }) + gate <- struct{}{} + waitFor("round 2", func() bool { return len(b.pending) == 0 }) + + // An op whose caller is slow to wake: nothing reads its done + slow := &op{run: ok, done: make(chan result, 2)} + b.mu.Lock() + b.pending = append(b.pending, slow) + b.mu.Unlock() + gate <- struct{}{} // the leader ends its turn with slow waiting + waitFor("the turn to end", func() bool { return len(slow.done) == 1 }) + + // A caller arriving before slow wakes up + got := make(chan bool, 1) + go func() { got <- b.do(ok) }() + select { + case r := <-got: + if !r { + t.Error("The new caller's write failed") + } + case <-time.After(5 * time.Second): + t.Fatal("The new caller waited for the woken op to take the lead") + } + if r := slow.outcome(); !r { + t.Error("The slow op's write failed") + } +} + // oneRound runs first and, once it leads a round blocked on the file mutex, // starts rest, so that every call of rest commits in the one round after // first's. It returns each call's result, first's at index 0. From 92446d185b9bd1b9a967a3cadacc7efe9bf2e055 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 16:39:56 +0600 Subject: [PATCH 4/4] fix: convert enqueued items in the caller, not in the round's leader database/sql converts each argument inside Exec, and Exec now runs in whichever goroutine leads the round. An item whose driver.Valuer panicked there unwound past the batcher, so the lead stayed taken, the round's other callers never got a result, and a grouped round left its write transaction open. Enqueue now converts the item with the default parameter converter, as database/sql would for this driver, before the op joins a round, so such a panic stays with its own caller. Tests: a panicking item panics in its caller and later writes still work; a committing grouped round gives each op its own result from one transaction; the batching test queues writes behind a held round instead of counting rounds under a racing scheduler. --- commit.go | 5 ++- commit_test.go | 119 ++++++++++++++++++++++++++++++++++++++++++++++--- queue.go | 12 ++++- queue_test.go | 6 +-- 4 files changed, 128 insertions(+), 14 deletions(-) diff --git a/commit.go b/commit.go index 429af8b..edb4f29 100644 --- a/commit.go +++ b/commit.go @@ -5,7 +5,10 @@ import ( "sync" ) -// op is one caller's write waiting for a round of the group commit +// op is one caller's write waiting for a round of the group commit. run +// executes in whichever goroutine leads the round, so it must not call user +// code (such as a driver.Valuer): a panic there would strand the round and +// leave its transaction open. Convert user values before building it. type op struct { run func(tx *sql.Tx) (bool, error) // one caller's statement(s); a nil tx means autocommit done chan result // buffered 2: one lead wake-up and the outcome diff --git a/commit_test.go b/commit_test.go index 87a2f72..b29a954 100644 --- a/commit_test.go +++ b/commit_test.go @@ -3,6 +3,7 @@ package sqliteq import ( "bytes" "database/sql" + "database/sql/driver" "errors" "runtime" "strconv" @@ -324,6 +325,89 @@ func TestCommitRoundFallsBackPerOp(t *testing.T) { } } +// TestCommitRoundGroupedResults runs a round that commits: each op gets its +// own outcome, from one shared transaction. +func TestCommitRoundGroupedResults(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + want := []bool{true, false, true, true} + txs := make([]*sql.Tx, len(want)) + ops := make([]*op, len(want)) + for i, ok := range want { + ops[i] = &op{ + run: func(tx *sql.Tx) (bool, error) { + txs[i] = tx + return ok, nil + }, + done: make(chan result, 2), + } + } + q.f.commitRound(ops) + + for i, w := range want { + if n := len(ops[i].done); n != 1 { + t.Errorf("op %d got %d results, want 1", i, n) + continue + } + if r := <-ops[i].done; r.ok != w || r.lead { + t.Errorf("op %d: got %+v, want ok=%t", i, r, w) + } + } + for i, tx := range txs { + if tx == nil || tx != txs[0] { + t.Errorf("op %d did not run in the round's one transaction", i) + } + } +} + +// panicValuer is an item whose conversion for the driver panics. +type panicValuer struct{} + +func (panicValuer) Value() (driver.Value, error) { panic("panicValuer") } + +// TestEnqueuePanickingItem checks that an item panicking in its conversion +// panics in its own caller and leaves the file's group commit working. +func TestEnqueuePanickingItem(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + func() { + defer func() { + if recover() == nil { + t.Error("Enqueue of a panicking item did not panic") + } + }() + q.Enqueue(panicValuer{}) + }() + + done := make(chan error, 1) + go func() { + if !q.Enqueue([]byte("x")) { + done <- errors.New("Enqueue failed") + return + } + _, ok, ackID := q.DequeueWithAckId() + if !ok { + done <- errors.New("DequeueWithAckId failed") + return + } + if !q.Acknowledge(ackID) { + done <- errors.New("Acknowledge failed") + return + } + done <- nil + }() + select { + case err := <-done: + if err != nil { + t.Error(err) + } + case <-time.After(10 * time.Second): + t.Fatal("Writes after the panic did not return") + } +} + // TestCommitRoundOneOpAutocommits checks that a round of one op runs without // a transaction, and a round of more shares one. func TestCommitRoundOneOpAutocommits(t *testing.T) { @@ -351,17 +435,38 @@ func TestCommitRoundOneOpAutocommits(t *testing.T) { } } -// TestGroupCommitBatches checks that concurrent writers share rounds. +// TestGroupCommitBatches checks that writes queued behind a running round, +// adds and acks alike, all commit in the one round after it. func TestGroupCommitBatches(t *testing.T) { q, qs := openQueue(t) defer qs.Close() + enqueueN(t, q, 16) - before := q.f.rounds.Load() - ops := concurrentWriters(t, q) - rounds := q.f.rounds.Load() - before - t.Logf("%d writes in %d rounds", ops, rounds) - if rounds >= int64(ops) { - t.Errorf("Expected fewer rounds than %d writes, got %d", ops, rounds) + items, ackIDs := make([]any, 16), make([]string, 16) + if n, _, err := q.DequeueBatchWithAckIds(items, ackIDs); err != nil || n != 16 { + t.Fatalf("Claimed %d, %v; want 16", n, err) + } + + add := func() bool { return q.Enqueue([]byte("x")) } + rest := []func() bool{} + for i := 0; i < 15; i++ { + rest = append(rest, add) + } + for _, id := range ackIDs { + rest = append(rest, func() bool { return q.Acknowledge(id) }) + } + + // oneRound fails the test unless exactly 2 rounds ran + for i, ok := range oneRound(t, q.f, add, rest) { + if !ok { + t.Errorf("Call %d failed", i) + } + } + if n := countRows(t, q.f.db, "1"); n != 16 { + t.Errorf("Expected 16 rows (the adds), got %d", n) + } + if n := q.Len(); n != 16 { + t.Errorf("Expected 16 pending items, got %d", n) } } diff --git a/queue.go b/queue.go index b2a17c8..bdcfa9f 100644 --- a/queue.go +++ b/queue.go @@ -3,6 +3,7 @@ package sqliteq import ( "cmp" "database/sql" + "database/sql/driver" "fmt" "slices" "strconv" @@ -272,6 +273,13 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { return false } + // Convert here, as database/sql would inside Exec, so the item's own code + // (a driver.Valuer) runs in its caller and never in a round's leader + data, err := driver.DefaultParameterConverter.ConvertValue(item) + if err != nil { + return false + } + now := time.Now().UTC() return q.f.b.do(func(tx *sql.Tx) (bool, error) { insert := q.insert @@ -280,9 +288,9 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { } var err error if q.priority { - _, err = insert.Exec(item, now, now, priority) + _, err = insert.Exec(data, now, now, priority) } else { - _, err = insert.Exec(item, now, now) + _, err = insert.Exec(data, now, now) } return err == nil, err }) diff --git a/queue_test.go b/queue_test.go index a340992..bf66ab0 100644 --- a/queue_test.go +++ b/queue_test.go @@ -745,9 +745,8 @@ func TestConcurrentWritersOneFile(t *testing.T) { } // concurrentWriters runs 8 producers and 16 ackers against one batch popper -// and checks every job is delivered once and acked. It returns the number of -// writes that go through the group commit: every Enqueue and Acknowledge. -func concurrentWriters(t *testing.T, q *Queue) int { +// and checks every job is delivered once and acked. +func concurrentWriters(t *testing.T, q *Queue) { t.Helper() const producers, perProducer, ackers, batch = 8, 500, 16, 16 const total = producers * perProducer @@ -826,7 +825,6 @@ func concurrentWriters(t *testing.T, q *Queue) int { if n := countRows(t, q.f.db, "1"); n != 0 { t.Errorf("Expected no rows left, got %d", n) } - return 2 * total } // TestWritesTakeFileMutex holds the file's write mutex and checks that every