From bd53f3c8c76c042b6f76dc896672eeb1e8f08e8d Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 09:57:56 +0600 Subject: [PATCH 1/2] perf: one-statement claims, a partial pending index, ack ids from row ids Claims and pops are one UPDATE/DELETE ... RETURNING statement, sorted back into queue order. Queue order is id (priority, id for priority queues), not created_at. Opening a table drops the four v0.2 indexes and creates one partial index on pending rows. The ack id is the row id; acks match only processing rows, so garbage, v0.2 cuids and repeat acks return false. Statements are prepared once per handle, the pool keeps 8 idle connections, and lucsky/cuid is gone. --- README.md | 6 +- go.mod | 2 - go.sum | 4 - migration_test.go | 208 +++++++++++++++++++++ priority_queue.go | 153 +-------------- priority_queue_test.go | 62 +++++- queue.go | 416 ++++++++++++++++++++++++----------------- queue_test.go | 235 ++++++++++++++++++++++- registry.go | 7 +- 9 files changed, 748 insertions(+), 345 deletions(-) create mode 100644 migration_test.go diff --git a/README.md b/README.md index 5bb0d53..f989d3c 100644 --- a/README.md +++ b/README.md @@ -177,10 +177,10 @@ sqliteq does not import varmq. Batch pops and `EnqueueOnce` are being added for SQLiteQ uses a SQLite database to store queue items with the following schema: -- `id`: Unique identifier for each item (autoincrement primary key) +- `id`: Unique identifier for each item (autoincrement primary key). Items come out in `id` order, and a priority queue's in `priority`, then `id` order. The ack id `DequeueWithAckId` returns is this id in decimal - `data`: The serialized item data (stored as a JSON blob) - `status`: The status of the item ("pending", "processing", or "completed") -- `ack_id`: A unique ID for acknowledging processed items +- `ack_id`: Kept from v0.2 so old tables need no rebuild; no longer used - `created_at`: When the item was added to the queue - `updated_at`: When the item was last updated @@ -191,7 +191,7 @@ SQLiteQ uses a SQLite database to store queue items with the following schema: - The queue is optimized for efficient enqueue and dequeue operations that scale well with queue size - Operations leverage SQLite's indexing for logarithmic time complexity rather than true constant-time - SQLite's WAL (Write-Ahead Logging) mode is enabled for better concurrent access -- Proper indexing is set up on the status and creation time columns for efficient querying +- One partial index covers the pending items in queue order; opening a v0.2 table replaces its old indexes with it ## 👤 Author diff --git a/go.mod b/go.mod index 7299849..b6ff707 100644 --- a/go.mod +++ b/go.mod @@ -3,5 +3,3 @@ module github.com/goptics/sqliteq go 1.24 require github.com/mattn/go-sqlite3 v1.14.28 - -require github.com/lucsky/cuid v1.2.1 diff --git a/go.sum b/go.sum index da4d537..42e5bac 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,2 @@ -github.com/lucsky/cuid v1.2.1 h1:MtJrL2OFhvYufUIn48d35QGXyeTC8tn0upumW9WwTHg= -github.com/lucsky/cuid v1.2.1/go.mod h1:QaaJqckboimOmhRSJXSx/+IT+VTfxfPGSo/6mfgUfmE= github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= -github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= diff --git a/migration_test.go b/migration_test.go new file mode 100644 index 0000000..36ef6c4 --- /dev/null +++ b/migration_test.go @@ -0,0 +1,208 @@ +package sqliteq + +import ( + "database/sql" + "fmt" + "path/filepath" + "sort" + "testing" +) + +// The v0.2.3 schema, pasted from tag v0.2.3's queue.go initTable and +// priority_queue.go initPriorityColumn, so the fixture needs no git. +const ( + v023CreateTable = ` + CREATE TABLE IF NOT EXISTS %[1]s ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + data BLOB NOT NULL, + status TEXT NOT NULL, + ack_id TEXT UNIQUE, + ack BOOLEAN DEFAULT 0, + created_at TIMESTAMP, + updated_at TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS %[2]s ON %[1]s (status, created_at); + CREATE INDEX IF NOT EXISTS %[3]s ON %[1]s (status, ack); + CREATE INDEX IF NOT EXISTS %[4]s ON %[1]s (ack_id); + ` + v023AddPriority = "ALTER TABLE %s ADD COLUMN priority INTEGER NOT NULL DEFAULT 0" + v023PriorityIndex = "CREATE INDEX IF NOT EXISTS %s ON %s (priority ASC, created_at ASC)" +) + +// writeV023File builds a v0.2.3 database at path with one table: 5 rows, ids +// 1-5, where 2 and 4 are processing with cuid ack ids. created_at runs +// backwards, so only id order puts them back in insert order. +func writeV023File(t *testing.T, path, name string, priority bool) { + t.Helper() + db, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatalf("Failed to open database: %v", err) + } + defer db.Close() + + exec := func(query string, args ...any) { + t.Helper() + if _, err := db.Exec(query, args...); err != nil { + t.Fatalf("Fixture statement failed: %v\n%s", err, query) + } + } + + exec(fmt.Sprintf(v023CreateTable, + quoteIdent(name), + quoteIdent(name+"_status_idx"), + quoteIdent(name+"_status_ack_idx"), + quoteIdent(name+"_ack_id_idx"))) + if priority { + exec(fmt.Sprintf(v023AddPriority, quoteIdent(name))) + exec(fmt.Sprintf(v023PriorityIndex, quoteIdent(name+"_priority_idx"), quoteIdent(name))) + } + + ackIDs := map[int]string{2: "cjld2cjxh0000qzrmn831i7rn", 4: "cjld2cyuq0000t3rmniod1foy"} + for i := 1; i <= 5; i++ { + status, ackID := "pending", sql.NullString{} + if id, ok := ackIDs[i]; ok { + status, ackID = "processing", sql.NullString{String: id, Valid: true} + } + created := fmt.Sprintf("2025-01-01 00:00:%02d+00:00", 60-i) + exec(fmt.Sprintf("INSERT INTO %s (id, data, status, ack_id, ack, created_at, updated_at) VALUES (?, ?, ?, ?, 0, ?, ?)", quoteIdent(name)), + i, []byte(fmt.Sprintf("job-%d", i)), status, ackID, created, created) + } +} + +// indexNames lists the indexes SQLite holds for a table. +func indexNames(t *testing.T, db *sql.DB, table string) []string { + t.Helper() + rows, err := db.Query("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = ?", table) + if err != nil { + t.Fatalf("Failed to list indexes: %v", err) + } + defer rows.Close() + + var names []string + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + t.Fatalf("Failed to scan index name: %v", err) + } + names = append(names, n) + } + if err := rows.Err(); err != nil { + t.Fatalf("Failed to list indexes: %v", err) + } + sort.Strings(names) + return names +} + +func TestMigrateV02File(t *testing.T) { + for _, priority := range []bool{false, true} { + name := "jobs" + if priority { + name = "pjobs" + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "v023.db") + writeV023File(t, path, name, priority) + + // Opened twice: the migration must hold up, and change nothing, on reopen + for round := 1; round <= 2; round++ { + qs := mustNew(t, path, WithSingleFile()) + var ( + dequeue func() (any, bool) + length func() int + db *sql.DB + ) + if priority { + pq, err := qs.NewPriorityQueue(name) + if err != nil { + t.Fatalf("Round %d: failed to open v0.2.3 priority queue: %v", round, err) + } + dequeue, length, db = pq.Dequeue, pq.Len, pq.f.db + } else { + q, err := qs.NewQueue(name) + if err != nil { + t.Fatalf("Round %d: failed to open v0.2.3 queue: %v", round, err) + } + dequeue, length, db = q.Dequeue, q.Len, q.f.db + } + + want := []string{name + "_pending", "sqlite_autoindex_" + name + "_1"} + if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { + t.Errorf("Round %d: indexes = %v, want %v", round, got, want) + } + + if round == 1 { + // The two processing rows are pending again after recovery + if n := length(); n != 5 { + t.Errorf("Expected 5 pending jobs after opening, got %d", n) + } + } else { + for i := 1; i <= 5; i++ { + item, ok := dequeue() + if !ok { + t.Fatalf("Dequeue %d failed", i) + } + if got, want := string(item.([]byte)), fmt.Sprintf("job-%d", i); got != want { + t.Errorf("Dequeue %d = %q, want %q", i, got, want) + } + } + if _, ok := dequeue(); ok { + t.Error("Expected the queue empty after 5 dequeues") + } + } + + if err := qs.Close(); err != nil { + t.Fatalf("Round %d: failed to close: %v", round, err) + } + } + }) + } +} + +// A table first used as a queue and later opened as a priority queue gets +// its pending index rebuilt on (priority, id), and the reverse. +func TestPendingIndexFollowsKind(t *testing.T) { + path := filepath.Join(t.TempDir(), "kind.db") + + for _, priority := range []bool{false, true, false} { + qs := mustNew(t, path, WithSingleFile()) + var db *sql.DB + if priority { + pq, err := qs.NewPriorityQueue("jobs") + if err != nil { + t.Fatalf("Failed to open priority queue: %v", err) + } + db = pq.f.db + } else { + q, err := qs.NewQueue("jobs") + if err != nil { + t.Fatalf("Failed to open queue: %v", err) + } + db = q.f.db + } + + rows, err := db.Query(`SELECT name FROM pragma_index_info('jobs_pending') ORDER BY seqno`) + if err != nil { + t.Fatalf("Failed to read index columns: %v", err) + } + var cols []string + for rows.Next() { + var c string + if err := rows.Scan(&c); err != nil { + t.Fatalf("Failed to scan column: %v", err) + } + cols = append(cols, c) + } + rows.Close() + + want := "[id]" + if priority { + want = "[priority id]" + } + if fmt.Sprint(cols) != want { + t.Errorf("priority=%v: jobs_pending columns = %v, want %s", priority, cols, want) + } + if err := qs.Close(); err != nil { + t.Fatalf("Failed to close: %v", err) + } + } +} diff --git a/priority_queue.go b/priority_queue.go index c369143..734bab3 100644 --- a/priority_queue.go +++ b/priority_queue.go @@ -1,165 +1,24 @@ package sqliteq -import ( - "fmt" - "time" - - "github.com/lucsky/cuid" -) - -// PriorityQueue extends Queue with priority-based dequeuing +// PriorityQueue extends Queue with priority-based dequeuing: lower numbers +// come out first, and equal priorities in insert order type PriorityQueue struct { *Queue } // newPriorityQueue creates a new SQLite-based priority queue -func newPriorityQueue(f *file, tableName string, opts ...Option) (*PriorityQueue, error) { - baseQueue, err := newQueue(f, tableName, opts...) +func newPriorityQueue(f *file, name string, opts ...Option) (*PriorityQueue, error) { + q, err := newQueue(f, name, true, opts...) if err != nil { return nil, err } - pq := &PriorityQueue{ - Queue: baseQueue, - } - - // Add the priority column if it doesn't exist - if err := pq.initPriorityColumn(); err != nil { - return nil, fmt.Errorf("failed to initialize priority column: %w", err) - } - - return pq, nil -} - -// initPriorityColumn adds the priority column to the table if it doesn't exist -func (pq *PriorityQueue) initPriorityColumn() error { - exists, err := hasColumn(pq.f.db, pq.tableName, "priority") - if err != nil { - return err - } - - if !exists { - // Add priority column with default value 0 - _, err := pq.f.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN priority INTEGER NOT NULL DEFAULT 0", quoteIdent(pq.tableName))) - if err != nil { - return err - } - } - - // Create index on priority (ASC for lower numbers = higher priority) - _, err = pq.f.db.Exec(fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (priority ASC, created_at ASC)", quoteIdent(pq.tableName+"_priority_idx"), quoteIdent(pq.tableName))) - if err != nil { - return err - } - - return nil + return &PriorityQueue{Queue: q}, nil } // Enqueue adds an item to the queue with a specified priority // Lower priority numbers will be dequeued first (0 is highest priority) // Returns true if the operation was successful func (pq *PriorityQueue) Enqueue(item any, priority int) bool { - if pq.closed.Load() { - return false - } - - now := time.Now().UTC() - tx, err := pq.f.db.Begin() - - if err != nil { - return false - } - - defer func() { - if err != nil { - tx.Rollback() - } - }() - - _, err = tx.Exec( - fmt.Sprintf("INSERT INTO %s (data, status, created_at, updated_at, priority) VALUES (?, ?, ?, ?, ?)", quoteIdent(pq.tableName)), - item, "pending", now, now, priority, - ) - - if err != nil { - return false - } - - err = tx.Commit() - - return err == nil -} - -// dequeueInternal overrides the base dequeueInternal method to consider priority -func (pq *PriorityQueue) dequeueInternal(withAckId bool) (any, bool, string) { - if pq.closed.Load() { - return nil, false, "" - } - - tx, err := pq.f.db.Begin() - - if err != nil { - return nil, false, "" - } - - defer func() { - if err != nil { - tx.Rollback() - } - }() - - // Get the highest priority pending item (lower priority numbers come first) - var id int64 - var data []byte - row := tx.QueryRow(fmt.Sprintf( - "SELECT id, data FROM %s WHERE status = 'pending' ORDER BY priority ASC, created_at ASC LIMIT 1", - quoteIdent(pq.tableName), - )) - err = row.Scan(&id, &data) - if err != nil { - return nil, false, "" - } - - // Update the status to 'processing' with ack ID or remove directly if no ack ID - now := time.Now().UTC() - var ackID string - - if withAckId { - ackID = cuid.New() - - _, err = tx.Exec( - fmt.Sprintf("UPDATE %s SET status = 'processing', ack_id = ?, updated_at = ? WHERE id = ?", quoteIdent(pq.tableName)), - ackID, now, id, - ) - } else { - // remove the row if there is no ack - _, err = tx.Exec( - fmt.Sprintf("DELETE FROM %s WHERE id = ?", quoteIdent(pq.tableName)), - id, - ) - } - - if err != nil { - return nil, false, "" - } - - // Commit transaction - err = tx.Commit() - - if err != nil { - return nil, false, "" - } - - return data, true, ackID -} - -// Dequeue overrides the base Dequeue method to use priority-based dequeuing -func (pq *PriorityQueue) Dequeue() (any, bool) { - item, success, _ := pq.dequeueInternal(false) - return item, success -} - -// DequeueWithAckId overrides the base DequeueWithAckId method to use priority-based dequeuing -func (pq *PriorityQueue) DequeueWithAckId() (any, bool, string) { - return pq.dequeueInternal(true) + return pq.enqueue(item, priority) } diff --git a/priority_queue_test.go b/priority_queue_test.go index a52b928..a80c8fb 100644 --- a/priority_queue_test.go +++ b/priority_queue_test.go @@ -2,6 +2,7 @@ package sqliteq import ( "fmt" + "strconv" "testing" _ "github.com/mattn/go-sqlite3" @@ -76,7 +77,7 @@ func TestPriorityQueue(t *testing.T) { // Verify the item is completely removed from the database var count int - row := pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'completed' OR status = 'processing'", pq.tableName)) + row := pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'completed' OR status = 'processing'", pq.table)) err := row.Scan(&count) if err != nil { t.Errorf("Error checking items in database: %v", err) @@ -131,7 +132,7 @@ func TestPriorityQueue(t *testing.T) { } // Verify the item is completely removed from the database - row = pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", pq.tableName)) + row = pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", pq.table)) err = row.Scan(&count) if err != nil { t.Errorf("Error checking items in database: %v", err) @@ -178,7 +179,7 @@ func TestPriorityQueue(t *testing.T) { // Verify item is in processing status in database var count int - row := pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'processing'", pq.tableName)) + row := pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'processing'", pq.table)) err := row.Scan(&count) if err != nil { t.Errorf("Error checking processing items: %v", err) @@ -194,7 +195,7 @@ func TestPriorityQueue(t *testing.T) { } // Check that processing item is removed - row = pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'processing'", pq.tableName)) + row = pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'processing'", pq.table)) err = row.Scan(&count) if err != nil { t.Errorf("Error checking processing items: %v", err) @@ -251,7 +252,7 @@ func TestPriorityQueue(t *testing.T) { // Verify item is removed from database var count int - row := pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'completed' OR status = 'processing'", pq.tableName)) + row := pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'completed' OR status = 'processing'", pq.table)) err := row.Scan(&count) if err != nil { t.Errorf("Error checking items in database: %v", err) @@ -436,7 +437,7 @@ func TestPriorityQueueRemoveOnCompleteOption(t *testing.T) { // Since dequeueInternal with withAckId=false now deletes directly, // the item should be removed from the database immediately var count int - row := pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", pq.tableName)) + row := pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", pq.table)) err = row.Scan(&count) if err != nil { t.Errorf("Error checking items in database: %v", err) @@ -452,7 +453,7 @@ func TestPriorityQueueRemoveOnCompleteOption(t *testing.T) { } // Check that the item is still in the database with processing status - row = pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'processing'", pq.tableName)) + row = pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'processing'", pq.table)) err = row.Scan(&count) if err != nil { t.Errorf("Error checking processing items: %v", err) @@ -467,7 +468,7 @@ func TestPriorityQueueRemoveOnCompleteOption(t *testing.T) { } // Since removeOnComplete is true, the item should be removed from the database - row = pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", pq.tableName)) + row = pq.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", pq.table)) err = row.Scan(&count) if err != nil { t.Errorf("Error checking items in database: %v", err) @@ -577,3 +578,48 @@ func TestPriorityQueueReopen(t *testing.T) { } } } + +func TestPriorityOrderByID(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("q") + if err != nil { + t.Fatalf("Failed to create priority queue: %v", err) + } + + // Job i has priority i%3; ties come out in insert order + const n = 300 + for i := 0; i < n; i++ { + if !pq.Enqueue([]byte(strconv.Itoa(i)), i%3) { + t.Fatalf("Enqueue %d failed", i) + } + } + if _, err := pq.f.db.Exec(`UPDATE "q" SET created_at = datetime('2025-01-01', '-' || id || ' seconds')`); err != nil { + t.Fatalf("Failed to rewrite created_at: %v", err) + } + + var want []string + for p := 0; p < 3; p++ { + for i := p; i < n; i += 3 { + want = append(want, strconv.Itoa(i)) + } + } + + for i, v := range pq.Values() { + if got := string(v.([]byte)); got != want[i] { + t.Fatalf("Values()[%d] = %q, want %q", i, got, want[i]) + } + } + for i := range want { + item, ok, ackID := pq.DequeueWithAckId() + if !ok { + t.Fatalf("DequeueWithAckId %d failed", i) + } + if got := string(item.([]byte)); got != want[i] { + t.Fatalf("DequeueWithAckId %d = %q, want %q", i, got, want[i]) + } + if !pq.Acknowledge(ackID) { + t.Fatalf("Acknowledge %d failed", i) + } + } +} diff --git a/queue.go b/queue.go index e7d3431..266f620 100644 --- a/queue.go +++ b/queue.go @@ -1,49 +1,82 @@ package sqliteq import ( + "cmp" "database/sql" "fmt" + "slices" + "strconv" + "strings" "sync/atomic" "time" - "github.com/lucsky/cuid" _ "github.com/mattn/go-sqlite3" ) +// remainingCap bounds the pending count a batch pop reports +const remainingCap = 256 + // Queue implements the Queue interface using SQLite as the storage backend type Queue struct { f *file - tableName string + name, table string // table is quoteIdent(name) + priority bool + order string // "id" or "priority, id" removeOnComplete bool closed atomic.Bool + + // Prepared once per handle; they close with the file's pool + insert, claim, pop, ack, count, countCapped, values *sql.Stmt } -// newQueue creates a new SQLite-based queue and its table. Recovery runs -// from file.open, once per table per file. -func newQueue(f *file, tableName string, opts ...Option) (*Queue, error) { +// claimed is one row a claim moved to processing +type claimed struct { + id int64 + priority int + data []byte +} + +// newQueue creates a SQLite-based queue on its table: the table itself, the +// index migration and the prepared statements. Recovery runs from file.open, +// once per table per file. +func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, error) { q := &Queue{ f: f, - tableName: tableName, + name: name, + table: quoteIdent(name), + priority: priority, + order: "id", removeOnComplete: true, // Default to removing completed items } + if priority { + q.order = "priority, id" + } // Apply any provided options for _, opt := range opts { opt(q) } + // The db is shared by every queue of the file, so it is not closed on errors if err := q.initTable(); err != nil { - // The db is shared by every queue of the file, so it is not closed here return nil, fmt.Errorf("failed to initialize table: %w", err) } + if err := q.migrate(); err != nil { + return nil, fmt.Errorf("failed to migrate indexes: %w", err) + } + if err := q.prepare(); err != nil { + return nil, fmt.Errorf("failed to prepare statements: %w", err) + } return q, nil } -// initTable initializes the queue table if it doesn't exist +// initTable creates the table as v0.2 defines it, so old tables need no +// rebuild, and adds the priority column to a priority queue's table when +// it is missing func (q *Queue) initTable() error { - createTableSQL := fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %[1]s ( + _, err := q.f.db.Exec(fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( id INTEGER PRIMARY KEY AUTOINCREMENT, data BLOB NOT NULL, status TEXT NOT NULL, @@ -51,38 +84,123 @@ func (q *Queue) initTable() error { ack BOOLEAN DEFAULT 0, created_at TIMESTAMP, updated_at TIMESTAMP - ); - CREATE INDEX IF NOT EXISTS %[2]s ON %[1]s (status, created_at); - CREATE INDEX IF NOT EXISTS %[3]s ON %[1]s (status, ack); - CREATE INDEX IF NOT EXISTS %[4]s ON %[1]s (ack_id); - `, - quoteIdent(q.tableName), - quoteIdent(q.tableName+"_status_idx"), - quoteIdent(q.tableName+"_status_ack_idx"), - quoteIdent(q.tableName+"_ack_id_idx")) - - _, err := q.f.db.Exec(createTableSQL) + )`, q.table)) + if err != nil || !q.priority { + return err + } + + exists, err := hasColumn(q.f.db, q.name, "priority") + if err != nil || exists { + return err + } + // Add priority column with default value 0 + _, err = q.f.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN priority INTEGER NOT NULL DEFAULT 0", q.table)) return err } -// requeueNoAck puts rows left in processing without an ack back to pending -func (q *Queue) requeueNoAck() error { - tx, err := q.f.db.Begin() +// migrate drops the v0.2 indexes and makes sure the one partial index on +// pending rows matches the queue order. On a migrated table it changes nothing. +func (q *Queue) migrate() error { + for _, suffix := range []string{"_status_idx", "_status_ack_idx", "_ack_id_idx", "_priority_idx"} { + if _, err := q.f.db.Exec("DROP INDEX IF EXISTS " + quoteIdent(q.name+suffix)); err != nil { + return err + } + } + + index := q.name + "_pending" + cols, err := indexColumns(q.f.db, index) if err != nil { return err } + if slices.Equal(cols, strings.Split(q.order, ", ")) { + return nil + } - _, err = tx.Exec( - fmt.Sprintf("UPDATE %s SET status = 'pending', updated_at = ? WHERE status = 'processing' AND ack = 0", - quoteIdent(q.tableName)), - time.Now().UTC(), - ) + // The table was last used as the other kind of queue + if len(cols) > 0 { + if _, err := q.f.db.Exec("DROP INDEX " + quoteIdent(index)); err != nil { + return err + } + } + _, err = q.f.db.Exec(fmt.Sprintf("CREATE INDEX %s ON %s (%s) WHERE status = 'pending'", quoteIdent(index), q.table, q.order)) + return err +} + +// indexColumns lists an index's columns in order; none if it does not exist +func indexColumns(db *sql.DB, index string) ([]string, error) { + rows, err := db.Query("SELECT name FROM pragma_index_info(?) ORDER BY seqno", index) if err != nil { - tx.Rollback() - return err + return nil, err } + defer rows.Close() - return tx.Commit() + var cols []string + for rows.Next() { + var col string + if err := rows.Scan(&col); err != nil { + return nil, err + } + cols = append(cols, col) + } + return cols, rows.Err() +} + +// prepare prepares every statement of the queue once. Each read keeps +// status = 'pending' literal, or SQLite cannot use the partial index. +func (q *Queue) prepare() (err error) { + defer func() { + if err != nil { + q.closeStmts() + } + }() + + cols, vals, returning := "data, status, ack, created_at, updated_at", "?, 'pending', 0, ?, ?", "id, data" + if q.priority { + cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data" + } + pending := fmt.Sprintf("SELECT id FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order) + + ack := fmt.Sprintf("DELETE FROM %s WHERE id = ? AND status = 'processing'", q.table) + if !q.removeOnComplete { + ack = fmt.Sprintf("UPDATE %s SET status = 'completed', ack = 1, updated_at = ? WHERE id = ? AND status = 'processing'", q.table) + } + + for _, s := range []struct { + stmt **sql.Stmt + query string + }{ + {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", q.table, cols, vals)}, + {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, + {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)}, + {&q.ack, ack}, + {&q.count, fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", q.table)}, + {&q.countCapped, fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE status = 'pending' LIMIT %d)", q.table, remainingCap)}, + {&q.values, fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY %s", q.table, q.order)}, + } { + if *s.stmt, err = q.f.db.Prepare(s.query); err != nil { + return err + } + } + + return nil +} + +// closeStmts closes the statements prepared so far +func (q *Queue) closeStmts() { + for _, s := range []*sql.Stmt{q.insert, q.claim, q.pop, q.ack, q.count, q.countCapped, q.values} { + if s != nil { + s.Close() + } + } +} + +// requeueNoAck puts rows left in processing without an ack back to pending +func (q *Queue) requeueNoAck() error { + _, err := q.f.db.Exec( + fmt.Sprintf("UPDATE %s SET status = 'pending', updated_at = ? WHERE status = 'processing' AND ack = 0", q.table), + time.Now().UTC(), + ) + return err } // hasColumn reports whether the table has a column with the given name @@ -113,187 +231,154 @@ func hasColumn(db *sql.DB, table, column string) (bool, error) { return false, rows.Err() } +// parseAckID turns an ack id back into its row id. Only the canonical decimal +// of a positive id is accepted, so a v0.2 cuid or any other spelling acks nothing. +func parseAckID(s string) (int64, bool) { + if s == "" || s[0] < '1' || s[0] > '9' { + return 0, false + } + id, err := strconv.ParseInt(s, 10, 64) + return id, err == nil && id > 0 +} + // Enqueue adds an item to the queue -// It serializes the item to JSON and stores it in the database // Returns true if the operation was successful func (q *Queue) Enqueue(item any) bool { + return q.enqueue(item, 0) +} + +// enqueue inserts one pending row; priority is stored on priority queues only +func (q *Queue) enqueue(item any, priority int) bool { if q.closed.Load() { return false } now := time.Now().UTC() - tx, err := q.f.db.Begin() - if err != nil { - return false - } - defer func() { - if err != nil { - tx.Rollback() - } - }() - - _, err = tx.Exec( - fmt.Sprintf("INSERT INTO %s (data, status, ack, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", - quoteIdent(q.tableName)), item, "pending", 0, now, now) - if err != nil { - return false + var err error + if q.priority { + _, err = q.insert.Exec(item, now, now, priority) + } else { + _, err = q.insert.Exec(item, now, now) } - - err = tx.Commit() return err == nil } -// dequeueInternal is a helper function for both Dequeue and DequeueWithAckId -// It handles the common operations of finding and retrieving an item from the queue -// If withAckId is true, it will generate and store an ack ID -func (q *Queue) dequeueInternal(withAckId bool) (item any, success bool, ackID string) { - if q.closed.Load() { - return nil, false, "" - } - - tx, err := q.f.db.Begin() +// claimN moves up to n pending rows to processing in one statement and +// returns them in queue order +func (q *Queue) claimN(n int) ([]claimed, error) { + rows, err := q.claim.Query(time.Now().UTC(), n) if err != nil { - return nil, false, "" + return nil, err } + defer rows.Close() - defer func() { + out := make([]claimed, 0, n) + for rows.Next() { + var c claimed + if q.priority { + err = rows.Scan(&c.id, &c.priority, &c.data) + } else { + err = rows.Scan(&c.id, &c.data) + } if err != nil { - tx.Rollback() + return nil, err } - }() - - // Get the oldest pending item - var id int64 - var data []byte - - // Only dequeue pending items in FIFO order - row := tx.QueryRow(fmt.Sprintf( - "SELECT id, data, ack_id FROM %s WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1", - quoteIdent(q.tableName), - )) + out = append(out, c) + } + // Reading to the end runs the statement to completion, which commits it + if err := rows.Err(); err != nil { + return nil, err + } - // Use NullString to handle NULL values from database - var nullAckID sql.NullString + // RETURNING order is unspecified + slices.SortFunc(out, func(a, b claimed) int { + if c := cmp.Compare(a.priority, b.priority); c != 0 { + return c + } + return cmp.Compare(a.id, b.id) + }) + return out, nil +} - // Scan the row data - err = row.Scan(&id, &data, &nullAckID) // ackID may be NULL for pending items - // Extract the string value if valid - if nullAckID.Valid { - ackID = nullAckID.String +// Dequeue removes and returns the next item from the queue +// Returns the item and a boolean indicating if the operation was successful +func (q *Queue) Dequeue() (any, bool) { + if q.closed.Load() { + return nil, false } + rows, err := q.pop.Query() if err != nil { - return nil, false, "" + return nil, false } + defer rows.Close() - // Update the status to 'processing' or delete the item, based on withAckId - now := time.Now().UTC() - - if withAckId { - if ackID == "" { - ackID = cuid.New() + var data []byte + found := false + for rows.Next() { + if err := rows.Scan(&data); err != nil { + return nil, false } - - // Update the item to processing status - _, err = tx.Exec( - fmt.Sprintf("UPDATE %s SET status = 'processing', ack_id = ?, updated_at = ? WHERE id = ?", - quoteIdent(q.tableName)), - ackID, now, id, - ) - } else { - // For regular Dequeue, just delete the item immediately - _, err = tx.Exec( - fmt.Sprintf("DELETE FROM %s WHERE id = ?", quoteIdent(q.tableName)), - id, - ) + found = true } + if rows.Err() != nil || !found { + return nil, false + } + return data, true +} - if err != nil { +// DequeueWithAckId moves the next item to processing and returns it with its +// acknowledgment ID, the row id in decimal +func (q *Queue) DequeueWithAckId() (any, bool, string) { + if q.closed.Load() { return nil, false, "" } - err = tx.Commit() - - if err != nil { + c, err := q.claimN(1) + if err != nil || len(c) == 0 { return nil, false, "" } - - return data, true, ackID -} - -// Dequeue removes and returns the next item from the queue -// Returns the item and a boolean indicating if the operation was successful -func (q *Queue) Dequeue() (any, bool) { - item, success, _ := q.dequeueInternal(false) - return item, success -} - -// DequeueWithAckId removes and returns the next item from the queue with an acknowledgment ID -// Returns the item, a boolean indicating if the operation was successful, and the acknowledgment ID -func (q *Queue) DequeueWithAckId() (any, bool, string) { - return q.dequeueInternal(true) + return c[0].data, true, strconv.FormatInt(c[0].id, 10) } -// Acknowledge marks an item as completed -// Returns true if the item was successfully acknowledged, false otherwise +// Acknowledge marks a processing item as completed, deleting it unless +// WithRemoveOnComplete(false) is set +// Returns true only if exactly one processing row had this ack ID func (q *Queue) Acknowledge(ackID string) bool { - tx, err := q.f.db.Begin() - if err != nil { + id, ok := parseAckID(ackID) + if !ok { return false } - var rowsAffected int64 - - defer func() { - if err != nil || rowsAffected == 0 { - tx.Rollback() - } - }() - - var result sql.Result + var ( + result sql.Result + err error + ) if q.removeOnComplete { - // If removeOnComplete is true, delete the acknowledged item - result, err = tx.Exec( - fmt.Sprintf("DELETE FROM %s WHERE ack_id = ? ", quoteIdent(q.tableName)), - ackID, - ) + result, err = q.ack.Exec(id) } else { - // Otherwise, mark it as completed and set ack to 1 (true in SQLite) - result, err = tx.Exec( - fmt.Sprintf("UPDATE %s SET status = 'completed', ack = 1, updated_at = ? WHERE ack_id = ?", quoteIdent(q.tableName)), - time.Now().UTC(), ackID, - ) + result, err = q.ack.Exec(time.Now().UTC(), id) } - if err != nil { return false } - rowsAffected, err = result.RowsAffected() - - if err != nil || rowsAffected == 0 { - return false - } - - err = tx.Commit() - - return err == nil + n, err := result.RowsAffected() + return err == nil && n == 1 } // Len returns the number of pending items in the queue func (q *Queue) Len() int { var count int - row := q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'pending'", quoteIdent(q.tableName))) - err := row.Scan(&count) - if err != nil { + if err := q.count.QueryRow().Scan(&count); err != nil { return 0 } return count } -// Values returns all pending items in the queue +// Values returns all pending items in the queue, in queue order func (q *Queue) Values() []any { - rows, err := q.f.db.Query(fmt.Sprintf("SELECT data FROM %s WHERE status = 'pending' ORDER BY created_at ASC", quoteIdent(q.tableName))) + rows, err := q.values.Query() if err != nil { return nil } @@ -316,22 +401,7 @@ func (q *Queue) Values() []any { // Purge removes all items from the queue func (q *Queue) Purge() { - tx, err := q.f.db.Begin() - if err != nil { - return - } - defer func() { - if err != nil { - tx.Rollback() - } - }() - - _, err = tx.Exec(fmt.Sprintf("DELETE FROM %s", quoteIdent(q.tableName))) - if err != nil { - return - } - - err = tx.Commit() + q.f.db.Exec(fmt.Sprintf("DELETE FROM %s", q.table)) } // Close closes the queue and its database connection diff --git a/queue_test.go b/queue_test.go index 99c534f..023bbdd 100644 --- a/queue_test.go +++ b/queue_test.go @@ -4,6 +4,9 @@ import ( "database/sql" "fmt" "path/filepath" + "strconv" + "sync" + "sync/atomic" "testing" ) @@ -72,7 +75,7 @@ func TestSQLiteQueue(t *testing.T) { // Verify the item is completely removed from the database var count int - row := q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'completed' OR status = 'processing'", q.tableName)) + row := q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'completed' OR status = 'processing'", q.table)) error := row.Scan(&count) if error != nil { t.Errorf("Error checking items in database: %v", error) @@ -175,7 +178,7 @@ func TestRemoveOnCompleteOption(t *testing.T) { // Verify the item is still in the database by checking directly var count int - row := q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'completed'", q.tableName)) + row := q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'completed'", q.table)) error := row.Scan(&count) if error != nil { t.Errorf("Error checking completed items: %v", error) @@ -210,7 +213,7 @@ func TestRemoveOnCompleteOption(t *testing.T) { // Since dequeueInternal with withAckId=false now deletes directly, // the item should be removed from the database immediately var count int - row := q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", q.tableName)) + row := q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", q.table)) error := row.Scan(&count) if error != nil { t.Errorf("Error checking items in database: %v", error) @@ -228,7 +231,7 @@ func TestRemoveOnCompleteOption(t *testing.T) { } // Check that the item is still in the database with processing status - row = q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'processing'", q.tableName)) + row = q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s WHERE status = 'processing'", q.table)) error = row.Scan(&count) if error != nil { t.Errorf("Error checking processing items: %v", error) @@ -243,7 +246,7 @@ func TestRemoveOnCompleteOption(t *testing.T) { } // Since removeOnComplete is true, the item should be removed from the database - row = q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", q.tableName)) + row = q.f.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", q.table)) error = row.Scan(&count) if error != nil { t.Errorf("Error checking items in database: %v", error) @@ -364,10 +367,230 @@ func TestRecoveryError(t *testing.T) { if err != nil { t.Fatalf("Failed to open database: %v", err) } - q := &Queue{f: &file{db: db}, tableName: "recovery"} + q := &Queue{f: &file{db: db}, name: "recovery", table: quoteIdent("recovery")} db.Close() if err := q.requeueNoAck(); err == nil { t.Error("Expected an error from requeueNoAck on a closed database") } } + +// openQueue opens queue "q" in a fresh folder store. +func openQueue(t *testing.T, opts ...Option) (*Queue, Queues) { + t.Helper() + qs := mustNew(t, t.TempDir()) + q, err := qs.NewQueue("q", opts...) + if err != nil { + qs.Close() + t.Fatalf("Failed to create queue: %v", err) + } + return q, qs +} + +// countRows counts the rows of table "q" matching where. +func countRows(t *testing.T, db *sql.DB, where string) int { + t.Helper() + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM "q" WHERE ` + where).Scan(&n); err != nil { + t.Fatalf("Failed to count rows: %v", err) + } + return n +} + +func TestAckIDIsRowID(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + q.Enqueue([]byte("a")) + q.Enqueue([]byte("b")) + q.Dequeue() + _, ok, ackID := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId failed") + } + + got, err := strconv.ParseInt(ackID, 10, 64) + if err != nil { + t.Fatalf("Ack id %q is not a decimal integer: %v", ackID, err) + } + var id int64 + if err := q.f.db.QueryRow(`SELECT id FROM "q" WHERE status = 'processing'`).Scan(&id); err != nil { + t.Fatalf("Failed to read the claimed row: %v", err) + } + if got != id || id != 2 { + t.Errorf("Ack id %d, claimed row id %d, want both 2", got, id) + } +} + +func TestAckGarbage(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + q.Enqueue([]byte("claimed")) + q.Enqueue([]byte("pending")) + _, ok, ackID := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId failed") + } + + // Garbage, a v0.2 cuid, non-canonical spellings of the claimed id, and + // the id of a row that is still pending + garbage := []string{"", "abc", "-1", "0", "99999999999999999999", "cjld2cjxh0000qzrmn831i7rn", + " " + ackID, "+" + ackID, "0" + ackID, ackID + ".0", "2"} + if ackID == "2" { + t.Fatalf("Expected the claimed row to be id 1, got ack id %q", ackID) + } + for _, id := range garbage { + if q.Acknowledge(id) { + t.Errorf("Acknowledge(%q) = true, want false", id) + } + } + + if n := countRows(t, q.f.db, "1"); n != 2 { + t.Errorf("Expected 2 rows untouched, got %d", n) + } + if n := countRows(t, q.f.db, "status = 'processing'"); n != 1 { + t.Errorf("Expected the claimed row still processing, got %d processing rows", n) + } + if !q.Acknowledge(ackID) { + t.Error("Acknowledge of the real id failed after the garbage") + } +} + +func TestAckTwice(t *testing.T) { + for _, remove := range []bool{true, false} { + t.Run(fmt.Sprintf("remove=%v", remove), func(t *testing.T) { + q, qs := openQueue(t, WithRemoveOnComplete(remove)) + defer qs.Close() + + q.Enqueue([]byte("a")) + _, ok, ackID := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId failed") + } + if !q.Acknowledge(ackID) { + t.Fatal("First Acknowledge failed") + } + if q.Acknowledge(ackID) { + t.Error("Second Acknowledge of the same id returned true") + } + }) + } +} + +func TestKeepCompleted(t *testing.T) { + q, qs := openQueue(t, WithRemoveOnComplete(false)) + defer qs.Close() + + q.Enqueue([]byte("done")) + q.Enqueue([]byte("waiting")) + _, ok, ackID := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId failed") + } + if !q.Acknowledge(ackID) { + t.Fatal("Acknowledge failed") + } + + if n := countRows(t, q.f.db, "status = 'completed' AND ack = 1"); n != 1 { + t.Errorf("Expected 1 completed row with ack = 1, got %d", n) + } + if n := q.Len(); n != 1 { + t.Errorf("Expected Len 1 (the completed row excluded), got %d", n) + } +} + +func TestOrderByID(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + const n = 1000 + for i := 0; i < n; i++ { + if !q.Enqueue([]byte(strconv.Itoa(i))) { + t.Fatalf("Enqueue %d failed", i) + } + } + // Enqueues inside one millisecond, or a clock stepping back, leave + // created_at out of insert order; here it runs backwards outright + if _, err := q.f.db.Exec(`UPDATE "q" SET created_at = datetime('2025-01-01', '-' || id || ' seconds')`); err != nil { + t.Fatalf("Failed to rewrite created_at: %v", err) + } + + values := q.Values() + if len(values) != n { + t.Fatalf("Expected %d values, got %d", n, len(values)) + } + for i, v := range values { + if got := string(v.([]byte)); got != strconv.Itoa(i) { + t.Fatalf("Values()[%d] = %q, want %q", i, got, strconv.Itoa(i)) + } + } + for i := 0; i < n; i++ { + item, ok := q.Dequeue() + if !ok { + t.Fatalf("Dequeue %d failed", i) + } + if got := string(item.([]byte)); got != strconv.Itoa(i) { + t.Fatalf("Dequeue %d = %q, want %q", i, got, strconv.Itoa(i)) + } + } +} + +func TestConcurrentClaimsNoDuplicates(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + const producers, perProducer, consumers = 8, 100, 8 + var wg sync.WaitGroup + for p := 0; p < producers; p++ { + wg.Add(1) + go func(p int) { + defer wg.Done() + for i := 0; i < perProducer; i++ { + if !q.Enqueue([]byte(fmt.Sprintf("%d-%d", p, i))) { + t.Errorf("Enqueue %d-%d failed", p, i) + } + } + }(p) + } + + var ( + mu sync.Mutex + seen = map[string]int{} + done atomic.Int64 + ) + var cw sync.WaitGroup + for c := 0; c < consumers; c++ { + cw.Add(1) + go func() { + defer cw.Done() + for done.Load() < producers*perProducer { + item, ok, ackID := q.DequeueWithAckId() + if !ok { + continue + } + mu.Lock() + seen[string(item.([]byte))]++ + mu.Unlock() + if !q.Acknowledge(ackID) { + t.Errorf("Acknowledge(%s) failed", ackID) + } + done.Add(1) + } + }() + } + wg.Wait() + cw.Wait() + + if len(seen) != producers*perProducer { + t.Errorf("Expected %d distinct jobs, got %d", producers*perProducer, len(seen)) + } + for job, n := range seen { + if n != 1 { + t.Errorf("Job %s delivered %d times", job, n) + } + } + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Errorf("Expected no rows left, got %d", n) + } +} diff --git a/registry.go b/registry.go index 9afd5ce..13f3652 100644 --- a/registry.go +++ b/registry.go @@ -89,6 +89,9 @@ func acquire(path string, pages int) (*file, error) { if err != nil { return nil, fmt.Errorf("sqliteq: failed to open database: %w", err) } + // Idle connections are kept, so none reopens, reruns the hook and re-prepares + db.SetMaxOpenConns(8) + db.SetMaxIdleConns(8) f := &file{ key: path, @@ -144,7 +147,7 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) return h, nil } // A closed handle is replaced on the same table, as first named - name = q.tableName + name = q.name } h := &handle{} @@ -155,7 +158,7 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) } h.pq = pq } else { - q, err := newQueue(f, name, opts...) + q, err := newQueue(f, name, false, opts...) if err != nil { return nil, err } From ac67005f47e173af606345c7db36607170735879 Mon Sep 17 00:00:00 2001 From: fahimfaisaal Date: Fri, 23 Oct 2026 11:10:27 +0600 Subject: [PATCH 2/2] fix: name the pending index so no queue name can collide with it SQLite keeps tables and indexes in one namespace. With the index named _pending, a single file could not hold queues orders and orders_pending: whichever opened second failed. The index is now __sqliteq_pending. --- migration_test.go | 40 +++++++++++++++++++++++++++++++++++++--- queue.go | 6 +++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/migration_test.go b/migration_test.go index 36ef6c4..affe32f 100644 --- a/migration_test.go +++ b/migration_test.go @@ -125,7 +125,7 @@ func TestMigrateV02File(t *testing.T) { dequeue, length, db = q.Dequeue, q.Len, q.f.db } - want := []string{name + "_pending", "sqlite_autoindex_" + name + "_1"} + want := []string{name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { t.Errorf("Round %d: indexes = %v, want %v", round, got, want) } @@ -180,7 +180,7 @@ func TestPendingIndexFollowsKind(t *testing.T) { db = q.f.db } - rows, err := db.Query(`SELECT name FROM pragma_index_info('jobs_pending') ORDER BY seqno`) + rows, err := db.Query("SELECT name FROM pragma_index_info(?) ORDER BY seqno", "jobs"+pendingIndexSuffix) if err != nil { t.Fatalf("Failed to read index columns: %v", err) } @@ -199,10 +199,44 @@ func TestPendingIndexFollowsKind(t *testing.T) { want = "[priority id]" } if fmt.Sprint(cols) != want { - t.Errorf("priority=%v: jobs_pending columns = %v, want %s", priority, cols, want) + t.Errorf("priority=%v: pending index columns = %v, want %s", priority, cols, want) } if err := qs.Close(); err != nil { t.Fatalf("Failed to close: %v", err) } } } + +// Tables and indexes share one namespace in SQLite, so the pending index +// must not take a name a queue could have: a file with queues "orders" and +// "orders_pending" opens in either order, and again on reopen. +func TestPendingIndexNameFreeForQueues(t *testing.T) { + for _, names := range [][]string{{"orders", "orders_pending"}, {"orders_pending", "orders"}} { + t.Run(names[0]+"_first", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared.db") + for round := 1; round <= 2; round++ { + qs := mustNew(t, path, WithSingleFile()) + for _, name := range names { + q, err := qs.NewQueue(name) + if err != nil { + t.Fatalf("Round %d: failed to open queue %q: %v", round, name, err) + } + want := fmt.Sprintf("%s-%d", name, round) + if !q.Enqueue(want) { + t.Fatalf("Round %d: enqueue to %q failed", round, name) + } + item, ok := q.Dequeue() + if !ok { + t.Fatalf("Round %d: dequeue from %q failed", round, name) + } + if got := string(item.([]byte)); got != want { + t.Errorf("Round %d: %q dequeued %q, want %q", round, name, got, want) + } + } + if err := qs.Close(); err != nil { + t.Fatalf("Round %d: failed to close: %v", round, err) + } + } + }) + } +} diff --git a/queue.go b/queue.go index 266f620..31fbd20 100644 --- a/queue.go +++ b/queue.go @@ -98,6 +98,10 @@ func (q *Queue) initTable() error { return err } +// pendingIndexSuffix names a queue's partial pending index. Tables and +// indexes share one namespace, so the suffix is one no queue name would take. +const pendingIndexSuffix = "__sqliteq_pending" + // migrate drops the v0.2 indexes and makes sure the one partial index on // pending rows matches the queue order. On a migrated table it changes nothing. func (q *Queue) migrate() error { @@ -107,7 +111,7 @@ func (q *Queue) migrate() error { } } - index := q.name + "_pending" + index := q.name + pendingIndexSuffix cols, err := indexColumns(q.f.db, index) if err != nil { return err