diff --git a/README.md b/README.md index 4ff8415..6b4c501 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,47 @@ -# SQLiteQ: A SQLite-Based Persistent Queue for Go - -[![Go Reference](https://img.shields.io/badge/go-pkg-00ADD8.svg?logo=go)](https://pkg.go.dev/github.com/goptics/sqliteq) -[![Go Report Card](https://goreportcard.com/badge/github.com/goptics/sqliteq)](https://goreportcard.com/report/github.com/goptics/sqliteq) -[![Go Version](https://img.shields.io/badge/Go-1.24+-00ADD8?style=flat-square&logo=go)](https://golang.org/doc/devel/release.html) -[![CI](https://github.com/goptics/sqliteq/actions/workflows/sqliteq.yml/badge.svg)](https://github.com/goptics/sqliteq/actions/workflows/go.yml) -[![codecov](https://codecov.io/gh/goptics/sqliteq/branch/main/graph/badge.svg)](https://codecov.io/gh/goptics/sqliteq) - -SQLiteQ is a persistent queue implementation in Go using SQLite as the storage backend. It provides efficient enqueue and dequeue operations and maintains persistence across application restarts. - -## Features - -- Efficient enqueue and dequeue operations -- Persistence via SQLite storage -- Support for acknowledgment-based processing -- Simple and clean API following the Queue interface -- Built as a persistence adapter for [varmq](https://github.com/goptics/varmq) +
+ + + + sqliteq + + +

+ Go Reference + CI + Codecov + Go Version + License +

+ +

SQLite-backed FIFO and priority queues for Go, with acknowledgments, batch pops, group commit and one file per queue.

+ +

+ Features · + Installation · + Quick Start · + Usage · + Storage · + Configuration · + Delivery Guarantees · + Upgrading · + Development · + Contributing +

+
+ +--- + +## ✨ Features + +- **FIFO and priority queues** stored in SQLite; a lower priority number is served first. +- **One file per queue** by default, so busy queues never wait on each other's write lock. `WithSingleFile()` keeps every queue in one file. +- **Acknowledgments**: items taken with an ack id that are never acknowledged go back to the queue when the file is next opened. +- **Batch dequeue** of many items in one SQL statement, each under its own ack id. +- **Group commit**: concurrent `Enqueue` and `Acknowledge` calls on one file share one transaction. +- **Deduplicated enqueue**: `EnqueueOnce` stores a job once per id while its row exists. +- **Automatic WAL checkpoints**, tunable with `WithCheckpointPages`. +- **Thread-safe**: share one queue object across goroutines. +- **Persistence adapter for [varmq](https://github.com/goptics/varmq)**. ## Installation @@ -22,106 +49,160 @@ SQLiteQ is a persistent queue implementation in Go using SQLite as the storage b go get github.com/goptics/sqliteq ``` -## Usage +Requires Go 1.24+ and cgo: sqliteq uses [mattn/go-sqlite3](https://github.com/mattn/go-sqlite3), so building it needs `CGO_ENABLED=1` and a C compiler. + +## Quick Start ```go package main import ( - "fmt" - "log" + "fmt" + "log" - "github.com/goptics/sqliteq" + "github.com/goptics/sqliteq" ) func main() { - // The parameter is a folder, created if missing. Each queue gets its own - // database file in it: data/my_queue.db, data/my_priority_queue.db, ... - queuesManager, err := sqliteq.New("data") - if err != nil { - log.Fatalf("Failed to open queues: %v", err) - } - defer queuesManager.Close() - - // Create a new queue - // The parameter is the name of the queue; it becomes the table and file name - queue, err := queuesManager.NewQueue("my_queue") - if err != nil { - log.Fatalf("Failed to create queue: %v", err) - } - - // You can also create a queue with custom options - // For example, to keep acknowledged items in the database: - queueWithOptions, err := queuesManager.NewQueue("my_other_queue", - sqliteq.WithRemoveOnComplete(false)) // Set to false to keep acknowledged items - if err != nil { - log.Fatalf("Failed to create queue: %v", err) - } - _ = queueWithOptions - - // Create a priority queue - priorityQueue, err := queuesManager.NewPriorityQueue("my_priority_queue") - if err != nil { - log.Fatalf("Failed to create priority queue: %v", err) - } - _ = priorityQueue - - // Enqueue items - queue.Enqueue([]byte("item 1")) - queue.Enqueue([]byte("item 2")) - - // Get queue length - fmt.Printf("Queue length: %d\n", queue.Len()) - - // Get all pending items - items := queue.Values() - fmt.Printf("All items: %v\n", items) - - // Simple dequeue - item, success := queue.Dequeue() - if success { - fmt.Printf("Dequeued item: %v\n", string(item.([]byte))) - } - - // Dequeue with acknowledgment - item, success, ackID := queue.DequeueWithAckId() - if success { - fmt.Printf("Dequeued item: %v with ack ID: %s\n", string(item.([]byte)), ackID) - - // Process the item... - - // Acknowledge the item after processing - acknowledged := queue.Acknowledge(ackID) - fmt.Printf("Item acknowledged: %v\n", acknowledged) - - // Note: By default, acknowledged items are removed from the database - // With WithRemoveOnComplete(false), they would be marked as completed instead - } - - // Purge the queue - queue.Purge() + qs, err := sqliteq.New("data") // a folder, created if missing + if err != nil { + log.Fatal(err) + } + defer qs.Close() + + queue, err := qs.NewQueue("jobs") // stored in data/jobs.db + if err != nil { + log.Fatal(err) + } + + queue.Enqueue([]byte("hello world")) // any value database/sql accepts; false for others + + if data, ok := queue.Dequeue(); ok { + fmt.Println(string(data.([]byte))) // items come back as []byte + } } ``` -`New` returns an error instead of panicking: for a path it cannot use, an invalid option, or a path that is an existing file when `WithSingleFile()` is not given. +## Usage -The same name always returns the same queue handle, in any letter case. Asking for `NewQueue("x")` after `NewPriorityQueue("x")` (or the reverse) is an error, and so is reaching a live queue again with different options. `Queue.Close` stops further `Enqueue` and `Dequeue` calls on that handle but still accepts `Acknowledge`, so jobs that are still running can finish. The file stays open until `Queues.Close`. +### Queue types -## Layout on disk +| Type | Constructor | Order | +| --------------- | --------------------------- | ------------------------------ | +| `Queue` | `qs.NewQueue(name)` | FIFO | +| `PriorityQueue` | `qs.NewPriorityQueue(name)` | Lowest first, FIFO among equal | -By default `path` is a folder and every queue lives in its own SQLite file, named after the queue in lower case, with SQLite's `-wal` and `-shm` files beside it: +Both types support the acknowledgment, dedup and batch APIs below. Basic operations: + +```go +queue.Enqueue([]byte("message")) +queue.Len() // pending items +queue.Values() // pending items, in queue order +queue.Purge() // delete every item +``` + +While a handle is open, the same name returns the same queue handle, in any letter case. Asking for `NewQueue("x")` after `NewPriorityQueue("x")` (or the reverse) is an error, and so is reaching a live queue again with different options. `Queue.Close` stops further `Enqueue` and `Dequeue` calls on that handle but still accepts `Acknowledge`, so jobs that are still running can finish. The file stays open until `Queues.Close`. + +### Priority queue + +```go +pq, err := qs.NewPriorityQueue("emails") +if err != nil { + log.Fatal(err) +} + +pq.Enqueue([]byte("newsletter"), 3) +pq.Enqueue([]byte("password reset"), 1) + +data, ok := pq.Dequeue() // "password reset" +``` + +### Acknowledgments + +```go +item, ok, ackID := queue.DequeueWithAckId() +if ok { + process(item) + queue.Acknowledge(ackID) +} +``` + +An item taken with `DequeueWithAckId` is marked as processing until it is acknowledged. The first time a queue is opened after its file is opened in the process, every item still marked as processing goes back to pending, so a job that was running when the process stopped runs again. There is no visibility timeout: unacknowledged items return only when the file is next opened, not when a closed queue is opened again on a file that is still open. + +By default an acknowledged item is deleted. With `WithRemoveOnComplete(false)` it is kept and marked as completed: + +```go +audit, err := qs.NewQueue("audit", sqliteq.WithRemoveOnComplete(false)) +``` + +### Enqueue once per job id + +`EnqueueOnce(item, id)` on a queue, and `EnqueueOnce(item, priority, id)` on a priority queue, add an item under an id unless a row with that id is already stored. A repeat stores nothing and returns true, so an add that is retried with the same id is stored once. + +```go +queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // stored +queue.EnqueueOnce([]byte("send invoice 42"), "invoice-42") // true, nothing stored + +pq.EnqueueOnce([]byte("rebuild index"), 1, "rebuild-index") +``` + +The id is remembered while its row exists: pending, processing, or completed with `WithRemoveOnComplete(false)`. `Dequeue` deletes the row, and so does the ack with the default remove-on-complete; after that the id can be added again. `Purge` forgets every id. An empty id behaves like `Enqueue`. + +### Batch dequeue + +`DequeueBatchWithAckIds` takes up to `min(len(items), len(ackIDs))` items in one statement, each under its own ack id. `remaining` is the number of pending items left after it, counted up to 256; `Len` gives the exact count. + +```go +items := make([]any, 8) +ackIDs := make([]string, 8) + +for { + n, remaining, err := queue.DequeueBatchWithAckIds(items, ackIDs) + for i := range n { + process(items[i]) + queue.Acknowledge(ackIDs[i]) + } + if err != nil || remaining == 0 { + break + } +} +``` + +### With varmq + +sqliteq implements varmq's persistent queue interfaces: + +```go +q, err := qs.NewQueue("jobs") +if err != nil { + log.Fatal(err) +} + +worker := varmq.NewWorker(func(j varmq.Job[string]) { + fmt.Println(j.Data()) +}) +defer worker.WaitAndStop() + +queue := worker.WithPersistentQueue(q) +queue.Add("hello") +``` + +sqliteq does not import varmq. Batch pops and `EnqueueOnce` need varmq ≥ v1.10.0. From that release, a worker whose only source is the sqliteq queue claims several jobs in one statement through `DequeueBatchWithAckIds` (a worker bound to several queues claims one job per call), and a job added with an id (`WithJobId`, or one from `WithJobIdGenerator`) goes through `EnqueueOnce`. An older varmq calls neither: it claims one job per call and adds every job with `Enqueue`, which still works. + +## Storage + +### Layout on disk + +By default `path` is a folder and every queue lives in its own SQLite file, named after the queue in lower case, with SQLite's `-wal` and `-shm` files beside it while the file is open (SQLite removes them when the last connection closes): ``` data/ -├── my_queue.db -├── my_queue.db-shm -├── my_queue.db-wal -├── my_other_queue.db -├── my_other_queue.db-shm -├── my_other_queue.db-wal -├── my_priority_queue.db -├── my_priority_queue.db-shm -└── my_priority_queue.db-wal +├── emails.db +├── emails.db-shm +├── emails.db-wal +├── jobs.db +├── jobs.db-shm +└── jobs.db-wal ``` Each file has its own connection pool, its own write lock and its own checkpoints, so busy queues do not wait on each other. `New` only creates the folder; a file is opened (and created) the first time you ask for its queue. @@ -131,71 +212,114 @@ Each file has its own connection pool, its own write lock and its own checkpoint `WithSingleFile()` makes `path` one database file that holds every queue as a table. This is the v0.2 layout, so a file written by v0.2 opens with this option and keeps its data: ```go -queuesManager, err := sqliteq.New("queue.db", sqliteq.WithSingleFile()) -if err != nil { - log.Fatalf("Failed to open queues: %v", err) -} -defer queuesManager.Close() +qs, err := sqliteq.New("queue.db", sqliteq.WithSingleFile()) ``` -If you upgrade from v0.2, add `WithSingleFile()` to keep reading your existing file, or start with the folder layout for new data. +The file is created if it is missing, but its folder must already exist: `New` returns an error otherwise, unlike the folder layout, which creates its folder. ### Queue names -In the folder layout the queue name becomes a file name, so it may only contain `A-Z a-z 0-9 . _ -`, must not be empty, and must not start with `sqlite_` (SQLite reserves that prefix). Anything else is an error from `NewQueue` / `NewPriorityQueue`. With `WithSingleFile()` any name works, since it is only a table name. +In the folder layout the queue name becomes a file name, so it may only contain `A-Z a-z 0-9 . _ -`, must not be empty, and must not start with `sqlite_` (SQLite reserves that prefix). Anything else is an error from `NewQueue` / `NewPriorityQueue`. On Windows the folder layout also cannot use a reserved device name as a queue name (`CON`, `PRN`, `AUX`, `NUL`, `COM1` to `COM9`, `LPT1` to `LPT9`, in any letter case), because the name would become a file name such as `con.db`. sqliteq does not check for these. With `WithSingleFile()` the name is only a table name, so any name SQLite accepts as a table name works, with two exceptions: SQLite reserves names that start with `sqlite_`, and since a queue `` creates the indexes `__sqliteq_pending` and `__sqliteq_job_id` (SQLite keeps tables and indexes in one namespace), a queue named like one of those cannot open in the same file as ``; whichever opens second gets an error. Names are case-insensitive in both layouts: `Jobs` and `jobs` are the same queue, stored in `jobs.db` in the folder layout. -## One process per file +### One process per file Run one process per database file. sqliteq keeps a registry in the process, keyed by the file's real path (made absolute, with symlinks resolved). Every `New` in the process that reaches the same file shares one connection pool and one write lock, so two `New` calls on one path never write through two competing pools. Two `Queues` values can therefore hold the same queue, and you get the same handle from both. The file closes when the last `Queues` using it is closed. -Another process is not coordinated with this one. When a queue opens, it puts every item still marked as processing back to pending, so a second process opening a file that is in use would rerun the first process's running jobs. sqliteq does not take an OS lock to stop that. +Another process is not coordinated with this one. The first time a process opens a queue, it puts every item still marked as processing back to pending, so a second process opening a file that is in use would rerun the first process's running jobs. sqliteq does not take an OS lock to stop that. + +On macOS, spell a path the same way every time: the registry key keeps the letter case you pass, so `Data` and `data` would be two entries for one folder. On Windows, Go resolves existing folders and files to their on-disk letter case, so different spellings of them share one entry. + +## Configuration + +| Option | Passed to | Default | Effect | +| ----------------------------- | ------------------------------ | ------------- | ---------------------------------------------------------------------- | +| `WithSingleFile()` | `New` | folder layout | `path` is one database file holding every queue | +| `WithCheckpointPages(n)` | `New` | `1000` | WAL size, in pages, at which SQLite checkpoints automatically | +| `WithRemoveOnComplete(false)` | `NewQueue`, `NewPriorityQueue` | `true` | Keep acknowledged items, marked as completed, instead of deleting them | -On case-insensitive filesystems (macOS, Windows), spell a path the same way every time: the registry key is case-sensitive, so `Data` and `data` would be two entries for one folder. +`New` returns an error instead of panicking: for a path it cannot use, an invalid option (`WithCheckpointPages` below 1, or a nil option), or a path that is an existing file when `WithSingleFile()` is not given. `NewQueue` and `NewPriorityQueue` return an error for a nil option too. -## Checkpoints +### Checkpoints SQLite appends writes to a write-ahead log (WAL) and folds them back into the database file in a checkpoint. `WithCheckpointPages(n)` sets how many pages the WAL may reach before SQLite checkpoints on its own. The default is 1000 pages, SQLite's own setting. A page is 4 KB, so a checkpoint starts once the WAL reaches about 4 MB. That size is where a checkpoint is triggered, not a hard limit on the file. A checkpoint runs inside the commit of the write that crosses that size, while that write holds the file's write lock. So with the default, any write on that file (an `Enqueue`, a claim, an ack or a purge) occasionally waits for a checkpoint's disk sync. ```go -queuesManager, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) +qs, err := sqliteq.New("data", sqliteq.WithCheckpointPages(10000)) ``` Raising it makes those waits rarer, which cuts `Enqueue` tail latency and raises throughput, at the price of a larger WAL on disk: 10000 pages triggers a checkpoint at about 41 MB. -In the `Mixed1Q` and `Drain1Q` rows of the [varmq-benchmarks `sqlite` benchmark](https://github.com/goptics/varmq-benchmarks/tree/main/sqlite) (one queue in one file, 20,000 jobs), its `walMB` metric, the largest size the WAL file reached, was 4.1 MB with 1000 pages and 41 MB with 10000 pages, measured on one machine. +The WAL is a log, not extra stored data: the database holds the same items either way. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. -The WAL is a log, not extra stored data: the database holds the same items either way. `n` must be at least 1, because `New` returns an error otherwise. Everything that reaches one file must use the same value: a `NewQueue` or `NewPriorityQueue` that reaches a file already open with a different value returns an error naming the value the file is open with. `New` itself does not open files, so it does not detect this. +## Delivery Guarantees -## Compatibility +| Call | Guarantee | If the process stops | +| -------------------------------------------- | --------------------------- | ---------------------------------------------------------- | +| `DequeueWithAckId`, `DequeueBatchWithAckIds` | At least once | Unacknowledged items go back to pending on the next open | +| `Dequeue` | At most once | The item is deleted when it is returned | +| `EnqueueOnce` | Once per id | Stored once while its row exists; the id survives recovery | +| `Enqueue` | Stored when it returns true | Survives a process crash | -sqliteq does not import varmq. Queues and priority queues implement `DequeueBatchWithAckIds`, so with varmq ≥ v1.10.0 a worker claims several jobs in one statement. An older varmq never calls it and claims one job per call, which still works. `EnqueueOnce` comes in a later release. +- Use `DequeueWithAckId` or `DequeueBatchWithAckIds` (varmq's persistent queue claims through one of them) for jobs you can't lose. +- Use globally unique ids with `EnqueueOnce` (UUID or nanoid, never a per-process counter). + +sqliteq runs SQLite in WAL mode with `synchronous=NORMAL`, unchanged from v0.2. A job whose `Enqueue` returned true survives a crash of the process. After a power loss or an operating system crash, the last commits before the failure may be lost, because SQLite rolls back what the WAL had not yet synced; the database itself stays consistent. + +## Upgrading from v0.2 + +- `New` returns `(Queues, error)` instead of panicking, and takes store options. `NewQueue` and `NewPriorityQueue` keep their signatures. +- `New(path)` now treats `path` as a folder with one file per queue. Add `WithSingleFile()` to keep reading an existing v0.2 file. +- `path` is a file system path, not a DSN. A `file:` URI or `:memory:` is taken as a literal file name and opens a new empty database; a `?` or `#` in the path is an error. Pass the plain path of a v0.2 file. +- While a queue is open, the same name returns the same handle instead of a new one, so `Queue.Close` on it closes it for every holder. Asking for it with different options, or as the other kind of queue, is an error. +- A v0.2 table is migrated when it opens: its old indexes are replaced with a partial index on pending rows, and a `job_id` column is added with a unique index over the rows that have one. Its data is kept. +- Ack ids are row ids. An ack id from v0.2 no longer matches and returns false. +- Items come out in `id` order (a priority queue's in `priority`, then `id`), not `created_at`. +- `RequeueNoAckRows` is no longer exported, and nothing replaces it: unacknowledged items are requeued the first time a queue is opened after its file is opened in the process, not on every `NewQueue` call. ## How It Works -SQLiteQ uses a SQLite database to store queue items with the following schema: +Each queue is a table with these columns: + +- `id`: autoincrement primary key. Items come out in `id` order, and a priority queue's in `priority`, then `id` order. The ack id `DequeueWithAckId` returns is this id in decimal. +- `priority`: priority queues only; a lower number comes out first. Opening an older table as a priority queue adds it. +- `data`: the stored item. +- `status`: `pending`, `processing` or `completed`. +- `ack`: 0 when an item is added, 1 once it is acknowledged and kept with `WithRemoveOnComplete(false)`. On open, only `processing` rows with `ack` 0 go back to pending. +- `ack_id`: kept from v0.2 so old tables need no rebuild; no longer used. +- `job_id`: the id an item was added with through `EnqueueOnce`, unique among the rows that have one; NULL for `Enqueue`. Opening an older table adds it. +- `created_at`, `updated_at`: when the item was added and last changed. + +On top of that table: + +- Two partial indexes: one covers the pending items in queue order, and a unique one covers `job_id` on the rows that have one, which `EnqueueOnce` relies on. +- A claim or a pop is one `UPDATE ... RETURNING` or `DELETE ... RETURNING` statement. +- Writes to one file (adds, claims, acks, purges) take one lock in the process, so they wait for each other in Go rather than in SQLite's busy handler. `Len` and `Values` do not take it. +- Concurrent `Enqueue` and `Acknowledge` calls on one file are committed together in one transaction. Each call still gets its own result, and if the shared transaction fails, each call is retried on its own so one bad call fails alone. + +## Development + +```bash +go test -race ./... # tests, with the race detector +go run ./example # a runnable example +gofmt -l . # formatting +``` + +Building and testing need cgo (`CGO_ENABLED=1`) and a C compiler. + +## API Reference + +See [GoDoc](https://pkg.go.dev/github.com/goptics/sqliteq). -- `id`: Unique identifier for each item (autoincrement primary key). Items come out in `id` order, and a priority queue's in `priority`, then `id` order. The ack id `DequeueWithAckId` returns is this id in decimal -- `data`: The serialized item data (stored as a JSON blob) -- `status`: The status of the item ("pending", "processing", or "completed") -- `ack_id`: Kept from v0.2 so old tables need no rebuild; no longer used -- `created_at`: When the item was added to the queue -- `updated_at`: When the item was last updated +## Star History -> NOTE: By default, when an item is acknowledged, it is removed from the database. However, you can configure the queue to keep acknowledged items by using the `WithRemoveOnComplete(false)` option when creating the queue. In this case, acknowledged items will be marked as "completed" but will remain in the database. +[![Star History Chart](https://api.star-history.com/svg?repos=goptics/sqliteq&type=Date)](https://www.star-history.com/#goptics/sqliteq&Date) -## Performance Considerations +## Contributing -- The queue is optimized for efficient enqueue and dequeue operations that scale well with queue size -- Operations leverage SQLite's indexing for logarithmic time complexity rather than true constant-time -- SQLite's WAL (Write-Ahead Logging) mode is enabled for better concurrent access -- One partial index covers the pending items in queue order; opening a v0.2 table replaces its old indexes with it -- Writes to one file (adds, claims, acks, purges) take one lock in the process, so they wait for each other in Go rather than in SQLite's busy handler. `Len` and `Values` do not take it -- A batch pop reports the items left after it, counting at most 256; `Len` gives the exact count +Open an issue to discuss a change, or send a pull request against `main` with tests (`go test -race ./...`). -## 👤 Author +## License -- GitHub: [@fahimfaisaal](https://github.com/fahimfaisaal) -- LinkedIn: [in/fahimfaisaal](https://www.linkedin.com/in/fahimfaisaal/) -- Twitter: [@FahimFaisaal](https://twitter.com/FahimFaisaal) +[MIT](LICENSE) diff --git a/assets/text-logo-dark.svg b/assets/text-logo-dark.svg new file mode 100644 index 0000000..f22b468 --- /dev/null +++ b/assets/text-logo-dark.svg @@ -0,0 +1,7 @@ + +sqliteq + + + +sqliteq + diff --git a/assets/text-logo.svg b/assets/text-logo.svg new file mode 100644 index 0000000..699c854 --- /dev/null +++ b/assets/text-logo.svg @@ -0,0 +1,7 @@ + +sqliteq + + + +sqliteq + diff --git a/commit_test.go b/commit_test.go index b29a954..6b1a7e1 100644 --- a/commit_test.go +++ b/commit_test.go @@ -296,7 +296,7 @@ func TestCommitRoundFallsBackPerOp(t *testing.T) { defer qs.Close() add := func(tx *sql.Tx) (bool, error) { - _, err := tx.Stmt(q.insert).Exec([]byte("x"), time.Now(), time.Now()) + _, err := tx.Stmt(q.insert).Exec([]byte("x"), time.Now(), time.Now(), nil) return err == nil, err } bad := func(*sql.Tx) (bool, error) { return false, errors.New("bad op") } diff --git a/dedup_test.go b/dedup_test.go new file mode 100644 index 0000000..35707c2 --- /dev/null +++ b/dedup_test.go @@ -0,0 +1,459 @@ +package sqliteq + +import ( + "fmt" + "path/filepath" + "sync" + "testing" +) + +// TestImplementsDedup checks at compile time that both kinds of queue +// implement varmq's IDedupQueue and IDedupPriorityQueue. +func TestImplementsDedup(t *testing.T) { + var _ interface{ EnqueueOnce(any, string) bool } = (*Queue)(nil) + var _ interface{ EnqueueOnce(any, int, string) bool } = (*PriorityQueue)(nil) +} + +// once adds item under id; on a priority queue it does so at priority 1 +type once func(item any, id string) bool + +// openOnce opens queue "q", or priority queue "q", in a fresh folder store, +// and returns its Queue with its EnqueueOnce. countRows reads table "q". +func openOnce(t *testing.T, priority bool, opts ...Option) (*Queue, once, Queues) { + t.Helper() + return openOnceAt(t, t.TempDir(), priority, opts...) +} + +// openOnceAt is openOnce in the folder store at dir, so a test can reopen it +func openOnceAt(t *testing.T, dir string, priority bool, opts ...Option) (*Queue, once, Queues) { + t.Helper() + qs := mustNew(t, dir) + if priority { + pq, err := qs.NewPriorityQueue("q", opts...) + if err != nil { + qs.Close() + t.Fatalf("Failed to create priority queue: %v", err) + } + return pq.Queue, func(item any, id string) bool { return pq.EnqueueOnce(item, 1, id) }, qs + } + q, err := qs.NewQueue("q", opts...) + if err != nil { + qs.Close() + t.Fatalf("Failed to create queue: %v", err) + } + return q, q.EnqueueOnce, qs +} + +// dequeueAll pops every pending item and returns them as strings, in order +func dequeueAll(t *testing.T, q *Queue) []string { + t.Helper() + var out []string + for { + item, ok := q.Dequeue() + if !ok { + return out + } + out = append(out, string(item.([]byte))) + } +} + +func TestEnqueueOnceRepeat(t *testing.T) { testOnceRepeat(t, false) } + +func testOnceRepeat(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("First EnqueueOnce failed") + } + if !add([]byte("b"), "j1") { + t.Error("A repeat returned false, want true") + } + if n := q.Len(); n != 1 { + t.Errorf("Expected 1 pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[a]" { + t.Errorf("Dequeued %v, want [a]", got) + } + + // Dequeue deleted the row, and the id with it + if !add([]byte("c"), "j1") { + t.Error("EnqueueOnce after Dequeue returned false, want true") + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c]" { + t.Errorf("Dequeued %v after the re-add, want [c]", got) + } +} + +func TestEnqueueOnceWhileProcessing(t *testing.T) { testOnceWhileProcessing(t, false) } + +func testOnceWhileProcessing(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { + t.Fatal("DequeueWithAckId failed") + } + + if !add([]byte("b"), "j1") { + t.Error("A repeat of a processing job returned false, want true") + } + if n := q.Len(); n != 0 { + t.Errorf("Expected no pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row (the processing one), got %d", n) + } +} + +func TestEnqueueOnceAfterAck(t *testing.T) { testOnceAfterAck(t, false) } + +func testOnceAfterAck(t *testing.T, priority bool) { + for _, remove := range []bool{true, false} { + t.Run(fmt.Sprintf("remove=%v", remove), func(t *testing.T) { + q, add, qs := openOnce(t, priority, WithRemoveOnComplete(remove)) + defer qs.Close() + + if !add([]byte("a"), "j1") { + t.Fatal("EnqueueOnce failed") + } + _, ok, ackID := q.DequeueWithAckId() + if !ok { + t.Fatal("DequeueWithAckId failed") + } + if !q.Acknowledge(ackID) { + t.Fatal("Acknowledge failed") + } + + if !add([]byte("c"), "j1") { + t.Error("EnqueueOnce after the ack returned false, want true") + } + if remove { + // The row is gone, and the id with it + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c]" { + t.Errorf("Dequeued %v, want [c]", got) + } + return + } + // The completed row keeps the id + if n := q.Len(); n != 0 { + t.Errorf("Expected no pending item, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row (the completed one), got %d", n) + } + }) + } +} + +// A job claimed and never acked is requeued on the next open, and its id +// still blocks a repeat: the job is stored once and runs once. +func TestEnqueueOnceAcrossRecovery(t *testing.T) { testOnceAcrossRecovery(t, false) } + +func testOnceAcrossRecovery(t *testing.T, priority bool) { + dir := t.TempDir() + q, add, qs := openOnceAt(t, dir, priority) + if !add([]byte("a"), "j1") { + qs.Close() + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { + qs.Close() + t.Fatal("DequeueWithAckId failed") + } + // The process "dies": the file closes with the row processing and no ack + if err := qs.Close(); err != nil { + t.Fatalf("Failed to close: %v", err) + } + + q, add, qs = openOnceAt(t, dir, priority) + defer qs.Close() + if n := q.Len(); n != 1 { + t.Fatalf("Len() = %d after reopen, want the claimed job back (1)", n) + } + + if !add([]byte("b"), "j1") { + t.Error("A repeat of the recovered job returned false, want true") + } + if n := countRows(t, q.f.db, "1"); n != 1 { + t.Errorf("Expected 1 row, got %d", n) + } + if n := countRows(t, q.f.db, "job_id = 'j1'"); n != 1 { + t.Errorf("Expected 1 row for j1, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[a]" { + t.Errorf("Dequeued %v, want [a] once", got) + } +} + +// Purge deletes the rows, so it frees their ids, pending and processing alike +func TestEnqueueOnceAfterPurge(t *testing.T) { testOnceAfterPurge(t, false) } + +func testOnceAfterPurge(t *testing.T, priority bool) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if !add([]byte("a"), "j1") || !add([]byte("b"), "j2") { + t.Fatal("EnqueueOnce failed") + } + if _, ok, _ := q.DequeueWithAckId(); !ok { // j1 is processing, j2 pending + t.Fatal("DequeueWithAckId failed") + } + + q.Purge() + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Fatalf("Expected no rows after Purge, got %d", n) + } + + if !add([]byte("c"), "j1") || !add([]byte("d"), "j2") { + t.Error("EnqueueOnce of a purged id returned false, want true") + } + if n := countRows(t, q.f.db, "1"); n != 2 { + t.Errorf("Expected 2 new rows, got %d", n) + } + if got := dequeueAll(t, q); fmt.Sprint(got) != "[c d]" { + t.Errorf("Dequeued %v, want [c d]", got) + } +} + +func TestEnqueueOncePriority(t *testing.T) { + t.Run("Repeat", func(t *testing.T) { testOnceRepeat(t, true) }) + t.Run("WhileProcessing", func(t *testing.T) { testOnceWhileProcessing(t, true) }) + t.Run("AfterAck", func(t *testing.T) { testOnceAfterAck(t, true) }) + t.Run("AcrossRecovery", func(t *testing.T) { testOnceAcrossRecovery(t, true) }) + t.Run("AfterPurge", func(t *testing.T) { testOnceAfterPurge(t, true) }) + + t.Run("Order", func(t *testing.T) { + qs := mustNew(t, t.TempDir()) + defer qs.Close() + pq, err := qs.NewPriorityQueue("q") + if err != nil { + t.Fatalf("Failed to create priority queue: %v", err) + } + + for _, c := range []struct { + item string + priority int + id string + }{ + {"low", 5, "a"}, + {"high", 1, "b"}, + {"mid", 3, "c"}, + {"high-again", 0, "b"}, // a repeat: neither stored nor moved up + {"low-again", 9, "a"}, + {"no-id", 2, ""}, + } { + if !pq.EnqueueOnce([]byte(c.item), c.priority, c.id) { + t.Fatalf("EnqueueOnce(%s, %d, %q) failed", c.item, c.priority, c.id) + } + } + + if got := dequeueAll(t, pq.Queue); fmt.Sprint(got) != "[high no-id mid low]" { + t.Errorf("Dequeued %v, want [high no-id mid low]", got) + } + }) +} + +// An empty id stores the item like Enqueue does, every time +func TestEnqueueOnceEmptyID(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + for i := 0; i < 3; i++ { + if !q.EnqueueOnce([]byte("x"), "") { + t.Fatalf("EnqueueOnce %d with no id failed", i) + } + if !q.Enqueue([]byte("y")) { + t.Fatalf("Enqueue %d failed", i) + } + } + if n := q.Len(); n != 6 { + t.Errorf("Expected 6 pending items, got %d", n) + } + if n := countRows(t, q.f.db, "job_id IS NULL"); n != 6 { + t.Errorf("Expected 6 rows with a NULL job_id, got %d", n) + } +} + +// The conflict clause covers the job_id index only: a nil item still fails, +// and a failed add leaves its id free +func TestEnqueueOnceNil(t *testing.T) { + for _, priority := range []bool{false, true} { + t.Run(fmt.Sprintf("priority=%v", priority), func(t *testing.T) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + if add(nil, "j1") { + t.Error("EnqueueOnce(nil) succeeded") + } + if q.Enqueue(nil) { + t.Error("Enqueue(nil) succeeded") + } + if n := countRows(t, q.f.db, "1"); n != 0 { + t.Errorf("Expected no rows, got %d", n) + } + if !add([]byte("a"), "j1") { + t.Error("EnqueueOnce after a failed one failed") + } + if n := q.Len(); n != 1 { + t.Errorf("Expected 1 pending item, got %d", n) + } + }) + } +} + +// Two adds of one id in one grouped round: the second conflicts inside the +// round's transaction, both return true, and one row is stored. With a nil +// item in the round, the round falls back to one transaction per call and +// the same holds. +func TestEnqueueOnceOneRound(t *testing.T) { + for _, bad := range []bool{false, true} { + t.Run(fmt.Sprintf("bad=%v", bad), func(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + first := func() bool { return q.Enqueue([]byte("x")) } + rest := []func() bool{ + func() bool { return q.EnqueueOnce([]byte("a"), "j1") }, + func() bool { return q.EnqueueOnce([]byte("b"), "j1") }, + func() bool { return q.EnqueueOnce([]byte("c"), "j2") }, + } + if bad { + rest = append(rest, + func() bool { return q.Enqueue(nil) }, + func() bool { return q.EnqueueOnce(nil, "j3") }, + ) + } + + got := oneRound(t, q.f, first, rest) + for i := 0; i < 4; i++ { + if !got[i] { + t.Errorf("Call %d failed", i) + } + } + for i := 4; i < len(got); i++ { + if got[i] { + t.Errorf("Call %d (a nil item) succeeded", i) + } + } + if n := countRows(t, q.f.db, "job_id = 'j1'"); n != 1 { + t.Errorf("Expected 1 row for j1, got %d", n) + } + if n := countRows(t, q.f.db, "1"); n != 3 { + t.Errorf("Expected 3 rows (x, j1, j2), got %d", n) + } + }) + } +} + +// 32 concurrent adds of one id: every call returns true and one row is stored +func TestEnqueueOnceConcurrent(t *testing.T) { + for _, priority := range []bool{false, true} { + t.Run(fmt.Sprintf("priority=%v", priority), func(t *testing.T) { + q, add, qs := openOnce(t, priority) + defer qs.Close() + + const n = 32 + var ( + wg sync.WaitGroup + start = make(chan struct{}) + oks [n]bool + ) + for g := 0; g < n; g++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + oks[g] = add([]byte(fmt.Sprint(g)), "j1") + }() + } + close(start) + wg.Wait() + + for g, ok := range oks { + if !ok { + t.Errorf("EnqueueOnce in goroutine %d returned false", g) + } + } + if l := q.Len(); l != 1 { + t.Errorf("Expected 1 pending item, got %d", l) + } + if c := countRows(t, q.f.db, "1"); c != 1 { + t.Errorf("Expected 1 row, got %d", c) + } + }) + } +} + +// A v0.2.3 file gains job_id and its unique partial index on open, keeps +// them on reopen, and its old rows still dequeue in order. +func TestEnqueueOnceOldFile(t *testing.T) { + for _, priority := range []bool{false, true} { + name := "jobs" + if priority { + name = "pjobs" + } + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "v023.db") + writeV023File(t, path, name, priority) + + for round := 1; round <= 2; round++ { + qs := mustNew(t, path, WithSingleFile()) + var ( + q *Queue + add once + ) + if priority { + pq, err := qs.NewPriorityQueue(name) + if err != nil { + t.Fatalf("Round %d: failed to open v0.2.3 priority queue: %v", round, err) + } + q, add = pq.Queue, func(item any, id string) bool { return pq.EnqueueOnce(item, 0, id) } + } else { + var err error + if q, err = qs.NewQueue(name); err != nil { + t.Fatalf("Round %d: failed to open v0.2.3 queue: %v", round, err) + } + add = q.EnqueueOnce + } + + if ok, err := hasColumn(q.f.db, name, "job_id"); err != nil || !ok { + t.Fatalf("Round %d: job_id column present = %v, %v", round, ok, err) + } + var unique, partial int + err := q.f.db.QueryRow(`SELECT "unique", partial FROM pragma_index_list(?) WHERE name = ?`, + name, name+jobIDIndexSuffix).Scan(&unique, &partial) + if err != nil || unique != 1 || partial != 1 { + t.Errorf("Round %d: job_id index unique=%d partial=%d err=%v, want a unique partial index", round, unique, partial, err) + } + if cols, err := indexColumns(q.f.db, name+jobIDIndexSuffix); err != nil || fmt.Sprint(cols) != "[job_id]" { + t.Errorf("Round %d: job_id index columns = %v, %v; want [job_id]", round, cols, err) + } + + if round == 1 { + if !add([]byte("new"), "j1") || !add([]byte("dup"), "j1") { + t.Error("EnqueueOnce on the old table failed") + } + if n := q.Len(); n != 6 { + t.Errorf("Expected 6 pending jobs, got %d", n) + } + } else { + want := "[job-1 job-2 job-3 job-4 job-5 new]" + if got := dequeueAll(t, q); fmt.Sprint(got) != want { + t.Errorf("Dequeued %v, want %s", got, want) + } + } + + if err := qs.Close(); err != nil { + t.Fatalf("Round %d: failed to close: %v", round, err) + } + } + }) + } +} diff --git a/layout_test.go b/layout_test.go index 0ff30a7..1b3bd0e 100644 --- a/layout_test.go +++ b/layout_test.go @@ -3,6 +3,7 @@ package sqliteq import ( "context" "database/sql" + "errors" "fmt" "os" "path/filepath" @@ -129,6 +130,66 @@ func TestNewBadPath(t *testing.T) { } } +// A nil option is an error from New, NewQueue and NewPriorityQueue, not a +// panic, and a rejected call leaves no folder, file or table behind +func TestNilOption(t *testing.T) { + t.Run("New", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "data") + for _, opts := range [][]StoreOption{{nil}, {WithSingleFile(), nil}} { + qs, err := New(dir, opts...) + if err == nil { + qs.Close() + t.Fatalf("Expected an error for a nil StoreOption in %d options", len(opts)) + } + if !errors.Is(err, errNilOption) { + t.Errorf("Got %v, want the nil option error", err) + } + } + if _, err := os.Stat(dir); err == nil { + t.Error("New created the folder before it rejected the option") + } + }) + + t.Run("NewQueue", func(t *testing.T) { + dir := t.TempDir() + qs := mustNew(t, dir) + defer qs.Close() + + if _, err := qs.NewQueue("q", nil); !errors.Is(err, errNilOption) { + t.Errorf("NewQueue with a nil option: got %v, want the nil option error", err) + } + if _, err := qs.NewPriorityQueue("p", nil); !errors.Is(err, errNilOption) { + t.Errorf("NewPriorityQueue with a nil option: got %v, want the nil option error", err) + } + if got := dbFiles(t, dir); len(got) != 0 { + t.Errorf("Expected no database file after the rejected calls, got %v", got) + } + + // The names are still free + if _, err := qs.NewQueue("q"); err != nil { + t.Errorf("Failed to create queue after the rejected call: %v", err) + } + if _, err := qs.NewPriorityQueue("p"); err != nil { + t.Errorf("Failed to create priority queue after the rejected call: %v", err) + } + }) + + // The registry rejects it before newQueue runs; newQueue's own check + // is the second line, and it must stop before it creates the table + t.Run("newQueue", func(t *testing.T) { + q, qs := openQueue(t) + defer qs.Close() + + if _, err := newQueue(q.f, "other", false, nil); !errors.Is(err, errNilOption) { + t.Errorf("newQueue with a nil option: got %v, want the nil option error", err) + } + var n int + if err := q.f.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name = 'other'`).Scan(&n); err != nil || n != 0 { + t.Errorf("Expected no table \"other\", got count %d, err %v", n, err) + } + }) +} + // dbFiles lists the database files in dir, leaving out SQLite's -wal and -shm files. func dbFiles(t *testing.T, dir string) []string { t.Helper() diff --git a/migration_test.go b/migration_test.go index affe32f..bc94204 100644 --- a/migration_test.go +++ b/migration_test.go @@ -125,7 +125,7 @@ func TestMigrateV02File(t *testing.T) { dequeue, length, db = q.Dequeue, q.Len, q.f.db } - want := []string{name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} + want := []string{name + jobIDIndexSuffix, name + pendingIndexSuffix, "sqlite_autoindex_" + name + "_1"} if got := indexNames(t, db, name); fmt.Sprint(got) != fmt.Sprint(want) { t.Errorf("Round %d: indexes = %v, want %v", round, got, want) } @@ -207,11 +207,12 @@ func TestPendingIndexFollowsKind(t *testing.T) { } } -// Tables and indexes share one namespace in SQLite, so the pending index -// must not take a name a queue could have: a file with queues "orders" and -// "orders_pending" opens in either order, and again on reopen. +// Tables and indexes share one namespace in SQLite, so the pending and +// job_id indexes must not take a name a queue could have: a file with queues +// "orders", "orders_pending" and "orders_job_id" opens in either order, and +// again on reopen. func TestPendingIndexNameFreeForQueues(t *testing.T) { - for _, names := range [][]string{{"orders", "orders_pending"}, {"orders_pending", "orders"}} { + for _, names := range [][]string{{"orders", "orders_pending", "orders_job_id"}, {"orders_job_id", "orders_pending", "orders"}} { t.Run(names[0]+"_first", func(t *testing.T) { path := filepath.Join(t.TempDir(), "shared.db") for round := 1; round <= 2; round++ { diff --git a/option.go b/option.go index ad485f6..c56379f 100644 --- a/option.go +++ b/option.go @@ -1,5 +1,11 @@ package sqliteq +import "errors" + +// errNilOption is returned by New, NewQueue and NewPriorityQueue for a nil +// option, which they would otherwise call +var errNilOption = errors.New("sqliteq: option must not be nil") + // Option is a function type that can be used to configure a Queue type Option func(*Queue) diff --git a/priority_queue.go b/priority_queue.go index bc2ff32..0aeae49 100644 --- a/priority_queue.go +++ b/priority_queue.go @@ -22,3 +22,9 @@ func newPriorityQueue(f *file, name string, opts ...Option) (*PriorityQueue, err func (pq *PriorityQueue) Enqueue(item any, priority int) bool { return pq.enqueue(item, priority, "") } + +// EnqueueOnce implements varmq.IDedupPriorityQueue: Queue.EnqueueOnce with a +// priority. A repeat leaves the stored item and its priority as they are. +func (pq *PriorityQueue) EnqueueOnce(item any, priority int, id string) bool { + return pq.enqueue(item, priority, id) +} diff --git a/queue.go b/queue.go index bdcfa9f..512555e 100644 --- a/queue.go +++ b/queue.go @@ -55,6 +55,9 @@ func newQueue(f *file, name string, priority bool, opts ...Option) (*Queue, erro // Apply any provided options for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(q) } @@ -80,6 +83,9 @@ func (q *Queue) initSchema() error { if err := q.migrate(); err != nil { return fmt.Errorf("failed to migrate indexes: %w", err) } + if err := q.addJobID(); err != nil { + return fmt.Errorf("failed to add job_id: %w", err) + } return nil } @@ -110,9 +116,13 @@ func (q *Queue) initTable() error { return err } -// pendingIndexSuffix names a queue's partial pending index. Tables and -// indexes share one namespace, so the suffix is one no queue name would take. -const pendingIndexSuffix = "__sqliteq_pending" +// pendingIndexSuffix and jobIDIndexSuffix name a queue's partial indexes. +// Tables and indexes share one namespace, so the suffixes are ones no queue +// name would take. +const ( + pendingIndexSuffix = "__sqliteq_pending" + jobIDIndexSuffix = "__sqliteq_job_id" +) // migrate drops the v0.2 indexes and makes sure the one partial index on // pending rows matches the queue order. On a migrated table it changes nothing. @@ -142,6 +152,24 @@ func (q *Queue) migrate() error { return err } +// addJobID adds the job_id column EnqueueOnce fills, when it is missing, and +// its unique index over the rows that have one. Rows without an id, old ones +// and those of Enqueue, are NULL and outside the index. +func (q *Queue) addJobID() error { + exists, err := hasColumn(q.f.db, q.name, "job_id") + if err != nil { + return err + } + if !exists { + if _, err := q.f.db.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN job_id TEXT", q.table)); err != nil { + return err + } + } + _, err = q.f.db.Exec(fmt.Sprintf("CREATE UNIQUE INDEX IF NOT EXISTS %s ON %s (job_id) WHERE job_id IS NOT NULL", + quoteIdent(q.name+jobIDIndexSuffix), q.table)) + return err +} + // indexColumns lists an index's columns in order; none if it does not exist func indexColumns(db *sql.DB, index string) ([]string, error) { rows, err := db.Query("SELECT name FROM pragma_index_info(?) ORDER BY seqno", index) @@ -170,7 +198,7 @@ func (q *Queue) prepare() (err error) { } }() - cols, vals, returning := "data, status, ack, created_at, updated_at", "?, 'pending', 0, ?, ?", "id, data" + cols, vals, returning := "data, status, ack, created_at, updated_at, job_id", "?, 'pending', 0, ?, ?, ?", "id, data" if q.priority { cols, vals, returning = cols+", priority", vals+", ?", "id, priority, data" } @@ -185,7 +213,9 @@ func (q *Queue) prepare() (err error) { stmt **sql.Stmt query string }{ - {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", q.table, cols, vals)}, + // The conflict target is the job_id index alone, so a NOT NULL or any + // other failure still errors; it must repeat the index's WHERE + {&q.insert, fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (job_id) WHERE job_id IS NOT NULL DO NOTHING", q.table, cols, vals)}, {&q.claim, fmt.Sprintf("UPDATE %s SET status = 'processing', updated_at = ? WHERE id IN (%s LIMIT ?) RETURNING %s", q.table, pending, returning)}, {&q.pop, fmt.Sprintf("DELETE FROM %s WHERE id IN (%s LIMIT 1) RETURNING data", q.table, pending)}, {&q.ack, ack}, @@ -266,8 +296,17 @@ func (q *Queue) Enqueue(item any) bool { return q.enqueue(item, 0, "") } +// EnqueueOnce implements varmq.IDedupQueue: it adds item under id unless a +// row with that id exists, pending, processing, or completed with +// WithRemoveOnComplete(false). A repeat stores nothing and returns true. An +// empty id behaves like Enqueue. +func (q *Queue) EnqueueOnce(item any, id string) bool { + return q.enqueue(item, 0, id) +} + // enqueue inserts one pending row in the file's next group commit round; -// priority is stored on priority queues only. id is unused for now. +// priority is stored on priority queues only. A row with a non-empty id is +// skipped, still with true, when a row with that id exists. func (q *Queue) enqueue(item any, priority int, id string) bool { if q.closed.Load() { return false @@ -280,18 +319,26 @@ func (q *Queue) enqueue(item any, priority int, id string) bool { return false } + // No id is NULL, which the job_id index leaves out; "" would be one id + var jobID any + if id != "" { + jobID = id + } + now := time.Now().UTC() return q.f.b.do(func(tx *sql.Tx) (bool, error) { insert := q.insert if tx != nil { insert = tx.Stmt(insert) } + // One statement, so a round of one, run without a transaction, is atomic var err error if q.priority { - _, err = insert.Exec(data, now, now, priority) + _, err = insert.Exec(data, now, now, jobID, priority) } else { - _, err = insert.Exec(data, now, now) + _, err = insert.Exec(data, now, now, jobID) } + // A repeat id changes no row and is no error: it succeeds return err == nil, err }) } diff --git a/queues.go b/queues.go index 490f63e..14836db 100644 --- a/queues.go +++ b/queues.go @@ -29,6 +29,9 @@ type Queues interface { func New(path string, opts ...StoreOption) (Queues, error) { cfg := defaultStoreConfig() for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(&cfg) } diff --git a/registry.go b/registry.go index 74d6151..24aeb0e 100644 --- a/registry.go +++ b/registry.go @@ -136,6 +136,9 @@ func (f *file) open(name string, priority bool, opts []Option) (*handle, error) want := Queue{removeOnComplete: true} for _, opt := range opts { + if opt == nil { + return nil, errNilOption + } opt(&want) }