From c9071451413b15d1f3d421cbc1aa61a56303d6d7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 19 Aug 2026 18:28:22 +0200 Subject: [PATCH] Promote the scheduler's work queue and command queue as the canonical implementation [release] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the contents of lib-data-workqueue, lib-data-workqueue-redis and lib-cmd-queue-redis with the lease-based implementation that has been running in the Seqera scheduler, which vendored these modules to iterate on them without a cross-repo release for every change. lib-data-workqueue 1.0.0 -> 2.0.0 lib-data-workqueue-redis 1.0.0 -> 2.0.0 lib-cmd-queue-redis 0.4.0 -> 1.0.0 PR #100 reverted the cmd-queue to 0.4.0 and left the two workqueue modules in the tree unpublished as placeholders, "because they carry the lease-based design forward". This fills those placeholders. Upstream held the names; the scheduler held the implementation. All three are breaking changes, so none reuses a burned coordinate: lib-cmd-queue-redis 0.5.0/0.5.1/0.6.0/0.7.0 and both workqueue modules' 1.0.0 are already published in the S3 repo, and publish.sh silently skips a version that exists — reusing one would go green and publish nothing. What changed in the design: - WorkQueue drops receive()/renewLease()/ack()/release() around a Lease record for init()/offer()/consume()/length(); consume() returns a Decision. - MessageConsumer returns Decision{ACK, RETRY, DEFERRED} instead of a boolean and receives a MessageLease settlement handle. New MessageLease interface: ack(), retry(), retryAfter(), bindLiveness(). - AbstractWorkQueue sheds the handler executor, semaphore slots, re-poll scheduler and heartbeat daemon; renewal moves into RedisWorkQueue, gated on the liveness supplier bound to each lease. Adds a cooperative drain — awaitQuiescent(), budgeted close(Duration), wait-once close(). - LocalWorkQueue moves to DelayQueue so a RETRY is paced rather than spun. - CommandServiceImpl replaces synchronous dispatch with lease-held execution, plus the hardening added while the scheduler ran it: per-command write mutex on CommandState, versioned CAS via VersionAware, retry-safe markProcessing(), rejection-safe submitCounted(), error tracking. Also here: - lib-data-workqueue(-redis) join the release step in build.yml. Their absence is how 1.0.0 reached S3 while missing from the list. - lib-cmd-queue-redis depends on lib-data-workqueue instead of lib-data-stream-redis, and exposes it plus lib-data-store-state-redis as api, since CommandQueue extends AbstractWorkQueue and CommandState implements VersionAware. - Dropped the lib-lang test dependency from both workqueue modules; no test imports io.seqera.lang. - Tests in lib-data-workqueue-redis move to io.seqera.data.workqueue.redis to match the main sources. The withdrawn design's tests (AsyncWorkQueue*Test, TunableQueue, TestWorkerPool) are deleted — they cannot compile against the new SPI; the drain, lease and config tests replace their coverage. - lib-data-stream-redis is untouched at 1.5.0 for the services that pin it, with a README pointer noting it is superseded and has no in-repo consumer left. Verified: 113 tests across the three modules, 0 failures; ./gradlew check green repo-wide except lib-util-net's SsrfValidatorTest, which needs live DNS and fails identically without this change. Generated POMs resolve to the bumped coordinates. Signed-off-by: Paolo Di Tommaso --- .github/workflows/build.yml | 2 + ...-07-31-command-state-write-mutex-design.md | 242 ++++++ ...mmand-execution-guarantee-message-lease.md | 705 ++++++++++++++++++ lib-cmd-queue-redis/README.md | 98 ++- lib-cmd-queue-redis/VERSION | 2 +- lib-cmd-queue-redis/build.gradle | 20 +- lib-cmd-queue-redis/changelog.txt | 34 + .../io/seqera/data/command/CommandConfig.java | 44 +- .../seqera/data/command/CommandHandler.java | 10 +- .../io/seqera/data/command/CommandQueue.java | 42 +- .../io/seqera/data/command/CommandResult.java | 6 +- .../seqera/data/command/CommandService.java | 40 + .../data/command/CommandServiceImpl.java | 624 +++++++++++----- .../io/seqera/data/command/CommandState.java | 57 +- .../io/seqera/data/command/CommandStatus.java | 20 +- .../data/command/store/CommandStateStore.java | 47 +- .../store/CommandStateStoreFactory.java | 2 +- .../command/store/CommandStateStoreImpl.java | 53 +- .../command/CommandQueueShowcaseTest.groovy | 14 +- ...mmandServiceCheckStatusIntervalTest.groovy | 89 +++ .../CommandServiceDrainBudgetTest.groovy | 150 ++++ .../command/CommandServiceDrainTest.groovy | 309 ++++++++ .../CommandServiceLeaseRedisTest.groovy | 451 +++++++++++ .../command/CommandServiceSafetyTest.groovy | 298 ++++++++ .../data/command/CommandServiceTest.groovy | 39 +- .../CommandStateSerializationTest.groovy | 56 +- .../store/CommandStateStoreRedisTest.groovy | 165 ++++ .../store/CommandStateStoreUpdateTest.groovy | 216 ++++++ .../data/command/StallingCommandQueue.java | 112 +++ .../data/command/TestCommandConfig.java | 17 +- .../seqera/data/command/TestCommandQueue.java | 6 +- .../src/test/resources/application-test.yml | 8 +- .../src/test/resources/logback-test.xml | 2 +- lib-data-stream-redis/README.md | 12 + lib-data-workqueue-redis/README.md | 107 ++- lib-data-workqueue-redis/VERSION | 2 +- lib-data-workqueue-redis/build.gradle | 1 - lib-data-workqueue-redis/changelog.txt | 14 + .../data/workqueue/redis/RedisWorkQueue.java | 562 +++++++++++--- .../workqueue/redis/RedisWorkQueueConfig.java | 71 +- .../workqueue/AsyncWorkQueueRedisTest.groovy | 208 ------ .../seqera/data/workqueue/TunableQueue.groovy | 82 -- .../AbstractWorkQueueRedisTest.groovy | 24 +- .../redis/RedisWorkQueueConfigTest.groovy | 121 +++ .../redis/RedisWorkQueueLeaseTest.groovy | 528 +++++++++++++ .../{ => redis}/RedisWorkQueueTest.groovy | 119 +-- .../workqueue/{ => redis}/TestConfig.groovy | 5 +- .../workqueue/{ => redis}/TestMessage.groovy | 2 +- .../workqueue/{ => redis}/TestQueue.groovy | 8 +- .../seqera/data/workqueue/TestWorkerPool.java | 36 - lib-data-workqueue/README.md | 184 ++--- lib-data-workqueue/VERSION | 2 +- lib-data-workqueue/build.gradle | 1 - lib-data-workqueue/changelog.txt | 24 + .../data/workqueue/AbstractWorkQueue.java | 543 +++++--------- .../seqera/data/workqueue/LocalWorkQueue.java | 169 ++++- .../data/workqueue/MessageConsumer.java | 135 ++-- .../seqera/data/workqueue/MessageLease.java | 92 +++ .../io/seqera/data/workqueue/WorkQueue.java | 186 ++--- .../metrics/MicrometerQueueMetrics.java | 109 ++- .../data/workqueue/metrics/Outcome.java | 6 +- .../data/workqueue/metrics/QueueMetrics.java | 39 +- .../AbstractWorkQueueDrainTest.groovy | 195 +++++ .../AbstractWorkQueueLocalTest.groovy | 42 +- .../AbstractWorkQueueMetricsTest.groovy | 145 +++- .../workqueue/AsyncWorkQueueLocalTest.groovy | 253 ------- .../data/workqueue/LocalWorkQueueTest.groovy | 257 ++++--- .../QueueMetricsClassloaderTest.groovy | 8 +- .../io/seqera/data/workqueue/TestQueue.groovy | 4 +- .../seqera/data/workqueue/TunableQueue.groovy | 82 -- .../seqera/data/workqueue/TestPlainQueue.java | 1 - .../seqera/data/workqueue/TestWorkerPool.java | 36 - 72 files changed, 6320 insertions(+), 2075 deletions(-) create mode 100644 docs/plans/2026-07-31-command-state-write-mutex-design.md create mode 100644 docs/plans/command-execution-guarantee-message-lease.md create mode 100644 lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceCheckStatusIntervalTest.groovy create mode 100644 lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceDrainBudgetTest.groovy create mode 100644 lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceDrainTest.groovy create mode 100644 lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceLeaseRedisTest.groovy create mode 100644 lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceSafetyTest.groovy create mode 100644 lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateStoreRedisTest.groovy create mode 100644 lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateStoreUpdateTest.groovy create mode 100644 lib-cmd-queue-redis/src/test/java/io/seqera/data/command/StallingCommandQueue.java delete mode 100644 lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/AsyncWorkQueueRedisTest.groovy delete mode 100644 lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TunableQueue.groovy rename lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/{ => redis}/AbstractWorkQueueRedisTest.groovy (70%) create mode 100644 lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueConfigTest.groovy create mode 100644 lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueLeaseTest.groovy rename lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/{ => redis}/RedisWorkQueueTest.groovy (51%) rename lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/{ => redis}/TestConfig.groovy (89%) rename lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/{ => redis}/TestMessage.groovy (95%) rename lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/{ => redis}/TestQueue.groovy (93%) delete mode 100644 lib-data-workqueue-redis/src/test/java/io/seqera/data/workqueue/TestWorkerPool.java create mode 100644 lib-data-workqueue/src/main/java/io/seqera/data/workqueue/MessageLease.java create mode 100644 lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueDrainTest.groovy delete mode 100644 lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AsyncWorkQueueLocalTest.groovy delete mode 100644 lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/TunableQueue.groovy delete mode 100644 lib-data-workqueue/src/test/java/io/seqera/data/workqueue/TestWorkerPool.java diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b85d8e6..ce87cba7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -78,6 +78,8 @@ jobs: bash publish.sh lib-data-store-future-redis bash publish.sh lib-data-store-state-redis bash publish.sh lib-data-stream-redis + bash publish.sh lib-data-workqueue + bash publish.sh lib-data-workqueue-redis bash publish.sh lib-fixtures-redis bash publish.sh lib-hashx bash publish.sh lib-lang diff --git a/docs/plans/2026-07-31-command-state-write-mutex-design.md b/docs/plans/2026-07-31-command-state-write-mutex-design.md new file mode 100644 index 00000000..c6f41161 --- /dev/null +++ b/docs/plans/2026-07-31-command-state-write-mutex-design.md @@ -0,0 +1,242 @@ +# Command State Write Mutex — Stop Losing Concurrent `CommandState` Writes + +Status: implemented — #895 (originally proposed here; #897 folded in, see §7) +Author: Paolo Di Tommaso +Date: 2026-07-31 +Stacked on: #890 (ordered shutdown / vendored command queue) + +--- + +## 1. Problem + +Every `CommandState` transition is a read-modify-write against Redis, and every write is +unconditional: + +```java +// CommandServiceImpl — 7 call sites +store.save(state.started()); +store.save(state.clearErrors()); +store.save(state.applyResult(result)); +store.save(state.withError(rootMessage(e))); // recordError +store.save(state.failed("No handler for type: " + state.type())); +store.save(state.cancelled()); // cancel() +``` + +`state` is a snapshot read earlier — in the dispatch path, potentially seconds earlier. Two writers +that read the same snapshot both write it back in full, so the later write silently discards the +earlier one. There is no version, no conditional write, and no lock. + +### 1.1 The case that matters + +`cancel()` reads the state, checks it is non-terminal, and writes `cancelled()`. Concurrently the +dispatcher can be finishing the same command and writing `applyResult(...)`. Whichever lands second +wins, so either: + +- the cancel is **silently swallowed** — the API returned `true`, the caller believes the command is + cancelled, and it completes anyway; or +- the terminal result is **overwritten by the cancel** — the command reports `CANCELLED` although + its work succeeded. + +The same shape applies between `recordError` / `clearErrors` and any concurrent transition, and it +gets worse once an execution can finish late (#890's follow-ups), because the gap between reading +`state` and writing it back widens from milliseconds to the whole handler duration. + +### 1.2 Why this was previously written off + +An earlier review concluded this was unfixable without a new store primitive. That was wrong. The +inherited API does lack a compare-and-swap: + +``` +StateStore: get / put / put+ttl / putIfAbsent / putIfAbsent+ttl / remove / clear +``` + +and `AbstractStateStore.delegate` is `private`, so a subclass cannot reach the provider. But +**`CommandStateStoreFactory` is handed the `StateProvider` directly**, and `CommandStateStoreImpl` +is vendored in this repo (#890), so it can keep its own reference to it. + +And CAS is not required: a safe read-modify-write needs **mutual exclusion**, and +`putIfAbsent(key, value, ttl)` + `remove(key)` is exactly that. + +## 2. Design + +Add one method to `CommandStateStore`, and make it the only way to mutate an existing state: + +```java +boolean update(String commandId, UnaryOperator mutator); +``` + +Implemented in `CommandStateStoreImpl` with a short-lived Redis mutex: + +1. `putIfAbsent(cmd-state/v1/lock:, token, stateLockTtl)` — fails fast if another writer holds it. +2. `get(id)`; return `false` if the command is gone or already terminal. +3. `put(id, mutator.apply(current))`. +4. `remove(lock)` in a `finally`. + +Contended acquisition retries briefly before giving up, so ordinary contention is invisible to +callers rather than surfacing as a failed write. + +Rejecting a transition from a terminal state is essential even after the re-read: the transition +methods themselves are unconditional, so allowing the mutator to run would merely serialize the +original last-writer-wins race. Once `SUCCEEDED`, `FAILED`, or `CANCELLED` is stored, no later +transition can replace it. + +### 2.1 Why a mutex is sufficient here, when a lease would not be + +The same primitive is unsafe for long critical sections and safe for short ones. This matters +because #890's follow-up work considers a *lease* over an entire handler execution, and the two +must not be conflated: + +| | Execution lease (not this change) | Write mutex (this change) | +|---|---|---| +| Critical section | the whole handler run — minutes for `ClusterCreateHandler` | one GET + one SET — sub-millisecond | +| TTL must exceed | unknown, unbounded handler duration | ~nothing | +| Expiry mid-section | plausible; needs refresh, and refresh cannot be fenced | needs a multi-second stop-the-world pause between two adjacent Redis calls | +| Cost of failure | duplicated cloud resources | one lost state write | + +So a lease spanning minutes without a fencing token is genuinely fragile, while a mutex spanning a +fraction of a millisecond with a seconds-long TTL is sound in practice. This change is only the +second column. + +### 2.2 Residual limitation + +This is a lock without fencing: if a writer stalls for longer than `stateLockTtl` *between* acquiring +and writing, a second writer can acquire and both proceed. That requires a GC pause longer than the +TTL landing between two adjacent Redis calls, and it costs a lost write rather than duplicated +infrastructure. + +The release is unconditional for the same reason: `StateProvider` has no compare-and-delete, so +`remove(lockKey)` cannot check ownership. A holder that stalled past the TTL therefore releases the +*next* holder's lock on its way out, letting a third writer in — the same stalled-past-TTL trigger +as above, one extra lost write in the worst case. The lock value is a per-acquisition token, so the +overlap is at least visible in diagnostics; conditioning release on it needs the same upstream +primitive as fencing. + +Making it airtight means adding a Lua-backed `replaceIf(key, expected, value)` to +`lib-data-store-state-redis` upstream. That is now a nice-to-have rather than a blocker, and is +deliberately out of scope. + +### 2.3 Making the safe path the only path + +`save()` is kept for the one genuine *create* (`submit()`), and its javadoc states that it +overwrites unconditionally and must not be used to transition an existing command. All seven +transition sites move to `update()`. Putting the lock inside the store rather than at the call +sites means a future transition cannot silently bypass it. + +## 3. Change list + +### `lib-cmd-queue-redis/.../store/CommandStateStore.java` +Add `update(String, UnaryOperator)`; document `save()` as create-only. + +### `lib-cmd-queue-redis/.../store/CommandStateStoreImpl.java` +Retain the `StateProvider`; implement `update()` as above; add `LOCK_PREFIX` and derive the retry +bound from configuration. + +### `lib-cmd-queue-redis/.../CommandConfig.java` +Add `stateLockTtl()` (5s) and `stateLockWait()` (100ms) as defaults, alongside the existing +`pollInterval` / `executeTimeout` / `stateTtl`. The lock lifetime and the total contended wait are +operator-tunable; the 20ms retry slice stays a private constant, and the attempt count is derived +from the wait — `attempts × interval` is one property and must not become two settings. + +`state-lock-wait` must stay well below `state-lock-ttl`: with `wait < ttl` a contended transition +fails fast and is retried from the queue, whereas with `wait >= ttl` the caller would block until a +stalled holder's lock expires, which is worse for a queue consumer. Documented, not enforced. + +### `sched-app` — `SchedCommandConfig` + `application.yml` +Expose both as `sched.command-queue.state-lock-{ttl,wait}`. + +### `lib-cmd-queue-redis/.../CommandServiceImpl.java` +Convert the seven transition sites. Failure semantics per site: + +| Site | On `update` == false | +|---|---| +| `cancel()` | return `false` — the caller is told the cancel did not take, instead of being told it did | +| terminal `applyResult` | log, `return false` — message stays queued; state was not written, so a retry is consistent | +| `started()` (both sites) | `markRunning()` (added by the #890 review) — retried once, then a loud warn unless the refusal was the terminal guard. RUNNING is what routes the next delivery to `checkStatus()`, so a silently missed mark re-runs `execute()`: against an execution abandoned by the timeout, or against an async job the handler already reported RUNNING | +| `clearErrors()` | ignore — best-effort bookkeeping | +| `recordError()` | ignore — already documented as best-effort | +| no-handler `failed()` | log, `return false` — retried rather than acked with unwritten state | + +## 4. Tests + +`CommandStateStoreUpdateTest`: + +1. **Concurrent updates do not lose a write** — two threads each increment `errorsCount`; assert the + final count is 2. This is the load-bearing test: it fails against blind `save()`. +2. `update` applies the mutator and persists it. +3. A terminal result cannot overwrite `CANCELLED`. +4. `CANCELLED` cannot overwrite a terminal result. +5. `update` returns `false` for an unknown command id. +6. The lock is released when the mutator throws (a following `update` still succeeds). +7. `update` returns `false` while another writer holds the lock. +8. `update` applies the mutator to the current value, not the caller's stale snapshot. + +`CommandServiceDrainTest` (§7): `drain()` stays inside its budget when the dispatcher never +quiesces — 500ms budget, asserts under 3s, where the unfixed path added a further 10s plus a 1s +join. + +## 5. Risks + +| Risk | Mitigation | +|---|---| +| Lock leaked by a crashed writer | TTL expiry; critical section is two Redis calls | +| Added latency per transition | one extra `SET NX` + one `DEL`; negligible against the work being reported | +| A transition site added later bypasses `update` | `save()` documented as create-only; the lock lives in the store, not at the call sites | +| Contention under load | bounded retry, then the caller keeps the message queued and retries — no lost work | + +## 6. Out of scope + +- The **execution lease** over a whole handler run (§2.1), which is where the fencing problem + actually bites. Separate decision, separate PR — it reintroduces leasing to a codebase that + deliberately reverted it. +- A Lua `replaceIf` upstream in `lib-data-store-state-redis` (§2.2). +## 7. Landed alongside (was out of scope, folded in via #897) + +The `drain()` → `close()` budget overlap from #890's review: `drain()` waited for the dispatcher and +then `close()` started a *fresh* `closeTimeout()`, so `drain(20s)` could take ~31s against a 25s +graceful-shutdown grace period and be hard-stopped mid-drain. `close(Duration)` now takes the +caller's budget and `drain()` passes what remains of its own, so there is one number rather than two +to keep aligned. + +Implementing that surfaced a further defect: with the budget exhausted, `close(0)` fell straight +through to `thread.interrupt()`, and that interrupt was observed reaching a handler still running on +the executor — the drain cutting short the work it exists to protect. `close()` therefore no longer +interrupts at all. The `closing` flag already guarantees the dispatcher exits at its next loop-head +check, and the thread is a daemon so it cannot hold up JVM exit, so an interrupt only shortened a +wait already abandoned while costing in-flight work and reviving the RESP-desync risk +(libseqera#92). + +`drain()` also preserves the result of the dispatcher's initial `awaitQuiescent()` call and reports +an incomplete drain when either the dispatcher is still inside synchronous `checkStatus()` work or +an executor-backed handler remains active. A blocked-`checkStatus()` regression test covers the +dispatcher-only case, which is not represented by the executor's in-flight counter. + +## 8. Hardened after the #890 review (`1c34b5683`, `3ef498651`) + +- **`markRunning()`** — the §3 `started()` row above: retried once on write-lock contention, + terminal-aware loud warn on a persistent miss. +- **`submitCounted()`** — the in-flight counter is incremented before `executor.submit()` and + decremented in the task's `finally`; a rejected submission (executor already torn down) now + decrements in a catch, since the task never runs. The two decrement sites are mutually exclusive + — an `execute()` throw surfaces via `Future.get()`, never out of `submit()` — so the count cannot + go down twice; without the catch a rejection leaks it upward for good and every later `drain()` + reports false. +- **`AbstractMessageStream.close()` waits only once.** After an explicit drain has spent (or given + up on) the cooperative wait, the `@PreDestroy` backstop's second `close()` returns immediately + instead of spending up to `closeTimeout()` again during bean destruction — the same + one-shutdown-budget argument as §7. +- **The graceful-shutdown delegate no longer blocks.** `CommandQueueGracefulShutdown` triggers the + drain via `runAsync` and returns the stage: the framework invokes delegates sequentially, so a + blocking drain serialized the Netty HTTP drain and the readiness flip behind it and made + `drain-timeout`, not the grace period, the effective bound. +- **`TaskSubmitHandler` permanent-throw guards** — with retry-on-throw active, a missing task row + returns `CommandResult.failure(...)` and a missing run row cancels the pending task via + `findRun()`; both were 60s poison loops when thrown. +- **Second-round polish** — the RUNNING-result `started()` site goes through `markRunning()` too + (it is the sole execute-once guard for a handler that reported an async job in flight); + `stateLockWait` default dropped 500ms → 100ms (the wait is consumed on the serial dispatcher + thread, and the critical section it guards is sub-millisecond); the acquire loop no longer + sleeps after its final attempt; the lock key moved to the `prefix + "/…"` secondary-key + namespace (`cmd-state/v1/lock:`) so it cannot collide with a state key by construction; + the lock value is a per-acquisition token rather than a per-store one; and the drain-incomplete + log names the dispatcher when `activeCommands()` reads zero, instead of reporting + "0 command(s) still running" during exactly the incident it exists for. diff --git a/docs/plans/command-execution-guarantee-message-lease.md b/docs/plans/command-execution-guarantee-message-lease.md new file mode 100644 index 00000000..cc827575 --- /dev/null +++ b/docs/plans/command-execution-guarantee-message-lease.md @@ -0,0 +1,705 @@ +# Command execution guarantee — message lease (PEL heartbeat) + task-settled delivery + +**Status**: **implemented and merged.** The lease, deferred settlement and task-settled delivery shipped in #913; the drain report and the shutdown refusal in #963 (which carried #964). Adversarially reviewed during design (22-agent review, 13 confirmed findings folded in — §Review log). +**How to read this**: §Requirements → §Guarantees describe the machinery as it exists in `CommandServiceImpl` and `RedisWorkQueue` today, and are kept in step with the code. §Lineage, §Alternative considered and §Review log are the archival design record and are deliberately not rewritten. +**Lineage**: selected over two alternatives explored during design — an execution-lock model (per-command lease key in the state store, rejected: a second ownership system beside the PEL, watchdog complexity, and it left crash recovery routed into handler semantics never promised) and a synchronous-dispatch variant of this same ownership model (kept as §Alternative) +**Modules touched**: `lib-data-workqueue` / `lib-data-workqueue-redis` (vendored), `lib-cmd-queue-redis` (vendored), `sched-app` (config + the `QueueMetrics` bean that wires the lease metrics into the DI context). No libseqera changes; no handler changes; no state-schema changes. +**Prerequisite**: the CAS `update()` in `CommandStateStoreImpl` (on this branch), which makes a refused state transition mean terminal-or-missing, never contention. + +## Requirements + +1. A handler may run longer than the visibility timeout without a second, overlapping + handler execution for the same command. +2. (Nearly) exactly-once execution. +3. Existing libseqera primitives only. **Handlers stay outside command state + management** — no recovery contracts, no audits, no new result types. + +## First principles + +Two design defects in the current model cause everything downstream: + +1. **The visibility timeout does two jobs.** It is the *failure detector* (dead consumer + → redeliver) and simultaneously the *upper bound on handler duration* (anything + slower is stolen mid-flight). This design removes the second job: a live + handler's message is heartbeated, so the visibility timeout degrades to its one + legitimate purpose — detecting a dead consumer. +2. **PROCESSING is dishonest.** Today the queue marks PROCESSING when it loses patience + (the 1s `execute-timeout`), not when the handler declared async work. That is + why crash recovery routes into `checkStatus()` semantics handlers never + promised to support (`ClusterDeleteHandler` polls a READY cluster forever; + `TaskLaunchBatchHandler` reports success unconditionally). Here PROCESSING only + ever means *"the handler returned PROCESSING"*: crash recovery is then simply + `execute()` again — the pre-existing #890 retry-on-throw contract, requiring + nothing new of handlers. + +The execution topology is **unchanged from today**: one dispatcher thread, handler +work on the blocking executor, `submitCounted` and `drain()` as they are. What +changes is who settles the message and when: **the handler task settles its own +entry when it finishes**, and the entry stays leased (heartbeated) for exactly as +long as the task runs. Ownership and execution have the same lifetime, held by the +same object — the PEL entry — so they cannot disagree. + +## Mechanism + +### 1. The PEL entry is the lease — give it a heartbeat + +Redis consumer-group ownership is per-entry mutual exclusion: an entry in a +consumer's PEL is invisible to `XREADGROUP >` and to `XAUTOCLAIM` below the idle +threshold. Its only flaw is the fixed idle clock. `XCLAIM` with `minIdle=0`, the +**same consumer**, and `JUSTID` resets the idle time without incrementing the +delivery counter (Jedis: `xclaimJustId`) — the "still alive, still mine" signal. + +`RedisWorkQueue` keeps a **queue-scoped** in-flight registry (composite +keying, mirroring `lastClaimCursor` — a bare `StreamEntryID` key would collide +across queues, since entry IDs are unique only per Redis stream key): + +```java +/** Leased entries per queue: registered before the consumer runs, removed on settle. */ +private final Map> inFlight = new ConcurrentHashMap<>(); +// Lease carries: registeredAt (max-age backstop), the settled flag, the metrics sample +``` + +One scheduled task per replica renews **per queue, in one round-trip,** at +`fixedRate = lease-renewal-period` (`sched.workqueue.lease-renewal-period`; +when unset it derives as `visibility-timeout / 4`, so the margin math below tracks a +re-tuned visibility timeout automatically — startup fails fast on a period at or +above the visibility timeout). The leak backstop is likewise configurable +(`sched.workqueue.max-lease-age`, derived default `3 × visibility-timeout`): + +```java +/** Scheduled at fixedRate = visibilityTimeout / 4. The tick body catches Throwable — + * an escaping Error would silently cancel a fixedRate task forever, losing every + * lease on the replica at once. */ +void renewLeases() { + final long start = System.nanoTime(); + try { + inFlight.forEach(this::renewQueue); + } + catch (Throwable t) { + log.error("Lease renewal tick failed", t); // outer wall; renewQueue contains per queue + } + finally { + final long elapsed = System.nanoTime() - start; + metrics.renewTick(elapsed, leasedCount()); + if (elapsed > periodNanos) { + log.warn("Lease renewal tick took {} — exceeds the renewal period; leases at risk", Duration.ofNanos(elapsed)); + } + } +} + +private void renewQueue(String queueId, Map leases) { + if (leases.isEmpty()) return; + try (Jedis jedis = pool.getResource()) { + // 1. Ownership check, one PIPELINED batch of per-id XPENDING calls (single + // round-trip, exact answers): an entry stolen during a renewal outage now + // belongs to another consumer. Renewing it blindly (minIdle=0 seizes + // regardless of owner) would re-steal it back mid-execution — ownership + // ping-pong. Instead: drop it, count it, log it. This makes the residual + // duplicate window OBSERVABLE instead of silent. + // INVARIANT: an id is renewed only when its ownership was positively + // confirmed this tick. A count-capped RANGE query is not safe here: leased + // ids bracketing more foreign pending entries than the cap get truncated out + // of the response, and treating "missing" as "safe to renew" re-seizes an + // entry another consumer legitimately owns. With per-id queries an absent id + // has exactly one meaning — no longer pending (acked/deleted): dropped, and + // counted as lost when the lease was never settled. + // 2. Age backstop, liveness-gated: a lease older than 3× the visibility timeout whose + // owner is not provably alive (MessageLease.bindLiveness — the dispatcher + // binds the handler task's Future) is a registry leak (a settlement path + // that never ran) — stop renewing, log loudly, let the claim cycle recover + // the entry. A leak can never be permanent, and a live handler is never + // age-pruned however long it runs (requirement 1) — an unconditional age + // prune would re-introduce the steal-mid-flight bug class at 3× the old + // threshold for any handler slower than 3×VT. + // 3. One variadic XCLAIM JUSTID for everything still ours — a single round-trip + // per queue per tick, so tick duration does not scale with in-flight count + // (25 sequential renewals against a slow-but-alive Redis would exceed the + // visibility timeout and lose every lease exactly when Redis degrades). + final List mine = checkOwnershipAndPrune(jedis, queueId, leases); // XPENDING + if (!mine.isEmpty()) { + jedis.xclaimJustId(queueId, config.getDefaultConsumerGroupName(), consumerName, + 0, new XClaimParams(), mine.toArray(StreamEntryID[]::new)); + } + } + catch (Exception e) { + // Transient: the next tick retries; see the margin math below. + metrics.renewError(); + log.warn("Lease renewal errored for queue {}, will retry", queueId, e); + } +} +``` + +**Margin math, stated honestly.** Writing `VT` for the visibility timeout: with +period `P = VT/4` (5s for the shipped +VT=20s) and a successful renewal at `t0`, the entry becomes claimable at +`t0 + VT`. Ticks fire at `t0+P, t0+2P, t0+3P` — so **two consecutive failed or +missed ticks are tolerated with a `VT/4` margin remaining**; the third strike +loses the lease. The safe outage tolerance is `VT − 2P − tick-latency` (~10s at +VT=20s), and the tick latency is bounded by design (two round-trips per queue, +not per entry). A replica crash wipes the in-memory registry → heartbeats stop → +the entry idles past the visibility timeout → redelivered elsewhere: exactly the +failure-detection semantics the visibility timeout was for. + +**Why VT=20s ships as the default.** The lease decouples the visibility timeout from +handler duration, so it is tuned purely for what it still governs: crash-recovery +latency and the transient-error retry cadence — both improve 3× versus the +previous 60s (a dead replica's work recovers, and a thrown handler retries, in +~20s). The PROCESSING re-poll cadence is deliberately NOT coupled to it: re-polls +pace on `check-status-interval` (45s — slightly tighter than the pre-lease 60s: +~25% faster discovery for ~33% more polls) via the delayed retry +below, so tightening crash detection does not multiply the `checkStatus()` load +on the DB/cloud reads every poll performs (with a PROCESSING `TaskSubmitHandler` +polling for the whole task lifetime, pollers ≈ in-flight tasks — PR #913 +review). The cost of the shorter visibility timeout is the narrower renewal-outage +tolerance above (~10s of Redis brownout instead of ~30s) before the residual +duplicate window re-opens — observable (`lease-lost` counter), damage bounded +by the CAS terminal write. Tune via `sched.workqueue.visibility-timeout` if +brownout tolerance ever matters more than recovery latency. + +**Delayed retry (`MessageLease.retryAfter`).** The visibility timeout would otherwise +still do two jobs — failure detection AND the re-poll cadence — the same +dual-role defect this design removed for handler duration. `retryAfter(delay)` +completes the decoupling: the lease stays registered (renewed, unstealable) +until `delay − VT` elapses, then the renewal tick releases it and the +natural idle-out delivers at ≈ the requested delay. A held lease is settled +(late `ack()`/`retry()` are no-ops), is deliberately excluded from the leak +backstop, and a delay at or below the visibility timeout degrades to a plain +`retry()` — the visibility timeout is the effective cadence floor. + +**Metrics** (all in the queue layer, Micrometer via the existing +`QueueMetrics` hook): leased-entries gauge, max-lease-age gauge, renewal-tick +timer, renewal-error counter, lease-lost counter (ownership check), lease-leak +counter (age backstop), and a renewal-liveness gauge (`lease.renewal.age`, +seconds since the last COMPLETED tick — the stuck-tick detector; alert above a +few renewal periods). + +**Bounded pool borrows (PR #913 review).** The Throwable containment above only +covers ticks that FAIL — a tick that BLOCKS is worse: the Jedis pool's +commons-pool2 default (`maxWait=-1`) waits indefinitely on an exhausted pool, so +a blocked borrow inside the tick never throws (no `renewError`, no overrun warn +— it lives in a `finally` that never runs) and the single-threaded +`scheduleAtFixedRate` never starts the next tick: every lease on the replica +goes stealable one visibility timeout later with zero telemetry, precisely during +the pool-exhausting brownout when leases matter most. Fixed at the source: +`lib-jedis-pool` 1.2.0 exposes `redis.pool.maxWait` and sched sets it to 2s — +below the 5s renewal period, so a starved tick fails loudly and the next one +runs on schedule (every borrower in this codebase is retry-safe). The +`lease.renewal.age` gauge is the belt for stuck-tick causes not yet imagined. `RedisWorkQueue` injects an *optional* +`QueueMetrics`, so production wiring is explicit: sched-app provides the bean +(`SchedCommandQueueFactory.queueMetrics`, present whenever a `MeterRegistry` +is) — without it every lease metric would silently be a no-op. + +### 2. Deferred settlement — the consumer API change + +The current contract (`boolean accept(msg)`: true = ack now, false = leave +pending) cannot express "a task now owns this entry". It becomes (named +`Decision`, not `Outcome` — the metrics package already owns that simple name): + +```java +public interface MessageConsumer { + + enum Decision { + ACK, // settle now: remove from the queue + RETRY, // leave pending: redelivered after the visibility timeout + DEFERRED // a task owns the lease; it will settle via MessageLease + } + + Decision accept(M message, MessageLease lease); + + /** Admission gate: the dispatcher does not claim from this queue while false. */ + default boolean ready() { + return true; + } +} + +/** Handle to settle a DEFERRED entry from any thread. Idempotent: first call wins. */ +public interface MessageLease { + void ack(); // stop renewal, then XACK + XDEL best-effort + void retry(); // stop renewal only; the entry re-claims after the visibility timeout + + /** Age-backstop gate: while the bound probe reports the owning task alive, the + * lease is never pruned as a leak. Default no-op (queues without renewal). */ + default void bindLiveness(BooleanSupplier alive) {} +} +``` + +Settlement rules the implementation must honor (review findings #4, #6, #9): + +- **Registration bracket**: `consume()` registers the entry *before* invoking + the consumer and un-registers on every path except a *returned* `DEFERRED` — + including a thrown exception, which settles as RETRY. Only the DEFERRED return + value transfers the lease to the task; nothing else may leave an entry + registered. +- **Un-register first, Redis second**: `ack()` flips the settled flag and + removes the entry from the registry *before* attempting `XACK`+`XDEL`. A + failed `XACK` then degrades to the benign case — the idle clock resumes, the + entry redelivers, and the delivery acks on the terminal check — instead of a + heartbeated-forever orphan. `retry()` is registry-removal only; **stopping + renewal is the release**, and the claim cadence is the retry schedule — no + Redis call at all. +- **Idempotence**: an `AtomicBoolean` on the lease; late `ack()`/`retry()` calls + are no-ops. `XCLAIM` without `FORCE` cannot resurrect an acked entry, so a + renewal racing an ack is harmless. (Note the scope of that claim: it covers + the *post-ack* race only. The *post-steal* case — another consumer now owns + the entry — is handled by the ownership check in `renewQueue`, not by XCLAIM + semantics.) + +`LocalWorkQueue` mirrors the semantics in-memory (a DEFERRED message is +unavailable until its lease settles; `retry()` re-queues it with a redelivery +delay (`workqueue.local.retry-delay`, default 1s) — the local analog of the claim +cadence, deliberately much shorter since local is a dev/test profile. Without +it, a consumer retrying fast (a handler repeatedly declaring PROCESSING, or +throwing quickly) would drive a hot loop of continuous redeliveries in +non-Redis deployments. The dispatcher reinforces the same pacing generically: +only ACK/DEFERRED count as progress for the idle-pause decision, so a +retry-only cycle sleeps the poll interval — on the Redis path this is harmless +(the entry is idle-gated anyway). The local provider still has no claim clock, +which tests must not +assume). sched is the only consumer of the vendored queue, so the API change +has exactly one call site to migrate. + +**Metrics mapping**: the claim-time sample travels on the lease; `ACK` records +`processed`, `RETRY` records `active`, a consumer throw records `errored` — at +decision time for synchronous decisions and at settle time for DEFERRED, so the +processing timer keeps measuring real handler duration. A `ready()==false` skip +records a `saturated` counter (not `EMPTY` — an admission-blocked replica under +backlog must be distinguishable from an idle one). + +### 3. Dispatch always — `executeWithTimeout` is deleted + +`CommandServiceImpl` submits every handler invocation to the blocking executor +and returns `DEFERRED` immediately; the dispatcher never waits on a handler. +The 1s budget, the abandoned-execution model, its discarded results, and the +patience-driven `markProcessing` are all deleted. + +```java +/** + * Queue consumer entry point. Deliveries settle three ways: ACK for stale/terminal + * messages, RETRY for transient refusals (redelivered after the visibility timeout), + * DEFERRED when a handler task takes the entry lease and settles it on completion. + * A throw out of this method is settled as RETRY by the queue layer. + */ +private MessageConsumer.Decision processCommand(CommandMsg msg, MessageLease lease) { + final var state = store.findById(msg.commandId()).orElse(null); + if (state == null) { + log.error("Command state not found - this should not happen: id={}", msg.commandId()); + return Decision.ACK; + } + if (state.status().isTerminal()) { + return Decision.ACK; + } + final var registration = getHandler(state.type()); + if (registration == null) { + log.error("No handler for command type: {}", state.type()); + return store.update(state.id(), s0 -> s0.failed("No handler for type: " + s0.type())) + ? Decision.ACK + : Decision.RETRY; + } + return dispatchCommand(state, registration, lease); +} + +/** + * Hand the delivery to a handler task. From a successful submit onward the TASK owns + * the lease — runCommand() settles it on every exit path. A rejected submit means + * nothing runs and nothing owns the lease: RETRY, the claim cycle re-delivers. + */ +private MessageConsumer.Decision dispatchCommand( + CommandState state, + CommandRegistration registration, + MessageLease lease) { + + final Command

command = toCommand(state, registration); // a throw here → RETRY via the queue layer + final CommandHandler handler = registration.handler(); + try { + final Future task = submitCounted(() -> runCommand(command, state, handler, lease)); + // the age backstop never prunes a live task's lease (see §1, backstop gate) + lease.bindLiveness(() -> !task.isDone()); + } + catch (RuntimeException e) { + log.error("Command dispatch rejected, will retry: id={}", state.id(), e); + return Decision.RETRY; + } + return Decision.DEFERRED; +} + +/** + * Runs on the blocking executor; the entry lease is renewed for as long as this + * takes, so a slow handler can never be stolen mid-flight. The finally guarantees + * every exit — including an Error out of handler code — settles the lease; the + * idempotent settle makes the happy-path ack and the finally's retry compose. + */ +private void runCommand(Command

command, CommandState state, + CommandHandler handler, MessageLease lease) { + try { + // Terminal snapshot: unreachable via processCommand()'s pre-check, but this method + // stays total rather than trusting its caller — the message is stale: ack. On its + // own branch BEFORE routing, so no handler result can ever be confused with it. + if (state.status().isTerminal()) { + lease.ack(); + return; + } + // Shutting down: this delivery was claimed before the shutdown began, and starting + // the handler now can only add work the drain has to wait out — one cloud call can + // carry a budget longer than the whole drain budget, so it cannot be waited out at + // all. Nothing is mutated at this point, so there is nothing to roll back and + // nothing to protect by proceeding. Settled as a retry: redelivered on the claim + // cadence — to a replica that is not shutting down, or to this process after a + // restart — and it runs from exactly this state. Checked AFTER the terminal branch, + // so a stale message is still acked here rather than pushed into the next process. + if (draining) { + log.debug("Command not started - service is draining: id={}, status={}", state.id(), state.status()); + lease.retry(); + return; + } + // Route explicitly on the snapshot status, every value named: reaching a terminal + // arm is a routing bug, and an unmapped new status must never silently execute — + // both throw, landing in the retry-on-throw catch below. + final CommandResult result = switch (state.status()) { + // PROCESSING is the handler's own earlier declaration (an execute() that returned + // PROCESSING) — never the queue's impatience — so checkStatus() is only invoked on + // the async-work pattern it was written for. + case PROCESSING -> handler.checkStatus(command, state); + // PENDING executes; after a crash the state is still PENDING, and the lease + // guarantees the crashed invocation is not still running on a live replica. + case PENDING -> handler.execute(command); + case SUCCEEDED, FAILED, CANCELLED -> throw new IllegalStateException("Terminal status must be acked before routing - id=" + state.id()); + default -> throw new IllegalStateException("Unmapped command status: " + state.status()); + }; + // A null result is a handler bug and must stay RETRYABLE — never a stale-message + // sentinel: acking would remove a live command's only message, stranding it with no + // retry driver left. + Objects.requireNonNull(result, () -> "Handler returned a null command result - id=" + state.id()); + + if (result.status() == CommandStatus.PROCESSING) { + // The handler declared async work in flight: record the declaration, then + // schedule the next checkStatus() poll on the re-poll cadence — decoupled from + // the visibility timeout, which paces crash detection and error retries. Tightening + // that clock must not multiply the polling load. + recordProcessingDeclaration(state); + lease.retryAfter(config.checkStatusInterval()); + return; + } + + // Terminal result, recorded via the CAS update: a refusal means the command went + // terminal underneath (a cancel won) — the redelivery acks on the terminal + // check, so RETRY loses nothing. + if (store.update(state.id(), s0 -> s0.applyResult(result))) { + log.debug("Command completed: id={}, status={}", state.id(), result.status()); + lease.ack(); + } + else { + // Two very different causes hide behind a refused terminal write and must not + // read as one: a VERIFIED terminal (or missing) state means a cancel or expiry + // won underneath — drop the result and ack the stale message now; anything else + // is the theoretical exhaustion of the CAS retry bound against a LIVE command — + // retried via redelivery, loudly. + settleUnrecordedResult(state, lease); + } + } + catch (Exception e) { + // Retry-on-throw (#890), now uniform: the state is unchanged, so the redelivery + // re-executes a PENDING command or re-polls a PROCESSING one. No rollback needed: + // there is no queue-invented PROCESSING to roll back. + log.error("Command processing errored, will retry: id={}", command.id(), e); + recordError(state, e); + lease.retry(); + } + finally { + // Backstop for non-Exception Throwables (OOME, NoClassDefFoundError out of + // handler code): a lease must never outlive its task. Idempotent — a no-op + // when a branch above already settled. + lease.retry(); + } +} +``` + +### 4. Admission cap — the throttle the 1s budget used to be + +Today the 1s dispatcher wait implicitly throttles claiming. With fire-and-submit +it must become explicit, or a backlog flood spawns unbounded tasks: + +```java +// CommandServiceImpl — MessageConsumer.ready() +@Override +public boolean ready() { + return inflight.size() < config.maxConcurrency(); +} +``` + +`inflight` is the in-flight **register** (§5), not a bare counter — its size *is* the +count, so the admission cap, `activeCommands()`, the drain wait and the shutdown +report cannot disagree with each other. `size()` on a concurrent map is an estimate +rather than a linearizable count, which costs nothing here: the only inserter is the +dispatcher thread that reads it, so it always observes its own insert, and the only +staleness comes from other threads' removals — which reads *high* and makes the cap +admit fewer, the safe direction for the pool the cap exists to protect. + +`AbstractWorkQueue.processMessages()` skips a queue whose consumer is not +`ready()` (recorded via the `saturated` counter; an idle loop still pauses on +the poll interval). Note the drain ceiling at saturation: a not-ready dispatcher +sleeps the FULL poll interval before re-checking, so backlog drain is additionally +bounded at ~`max-concurrency / poll-interval` admissions (~20/s at the defaults) +regardless of how fast handlers finish — raising the cap alone buys +proportionally less than it looks for fast handlers; lower `poll-interval` too +if backlog drain rate ever matters. New config +`sched.command-queue.max-concurrency` (default 20 — +sized below the shared JDBC pool minus request-path headroom, so handler bursts +queue in the work queue rather than starving the API/health probe of connections; +the pool was raised to 35 alongside, PR #913 review), +replacing `execute-timeout`, which is deleted. This closes the "no in-flight +admission cap" gap documented in `application.yml` since the #685 revert. + +### 5. The in-flight register, and what a drain reports (#963) + +`inflight` is a `Map` keyed by a **monotonic sequence**, valued by +`()`. Keying on the sequence rather than the command id matters more +than it looks: two invocations of one command in flight at once (not expected under the +lease, but not structurally impossible) must not collapse into a single entry — that +would under-count the admission cap and the drain, not merely lose a name. The entry is +registered in `submitCounted` before the task runs and unregistered on every exit path, +including a rejected executor submit; the two removal sites are mutually exclusive, so a +submission can never remove twice. + +`drain(Duration)` arms the shutdown signal, stops the dispatcher claiming +(`awaitQuiescent`), waits for the register to empty, then releases the queue — bounded +by what is left of the caller's budget, so `close()` cannot start a second timer of its +own and push the drain past the container's grace period. + +The first two waits **share** the budget, and the deadline for the second is taken before +the first runs — so whatever the dispatcher wait spends comes out of the register wait's +share. `quiesceBudget()` therefore caps the dispatcher wait at a quarter of the budget. +Handed all of it, a dispatcher that never quiesces leaves the register wait with nothing: +its loop runs zero iterations and handler tasks mid-flight against the database get no +grace at all, which is the one thing the drain exists to provide (#888). A quarter derives +from the caller's budget the way the lease renewal period derives from the visibility +timeout, so re-tuning `drain-timeout` scales both halves and there is no second dial to +keep in step. It is generous by construction: the dispatcher's own work between two +loop-head checks is a `findById` and a dispatch decision, since every handler invocation +runs on the executor and a delivery claimed after the shutdown began is not routed at all. + +The cap does not distort the report. `close()` waits again with the leftover budget, so a +dispatcher that outlives its quiesce budget but stops before the drain budget expires is +*slow, not stuck* — the drain re-probes after `close()` returns and reports success, with +a dedicated warning naming the exceeded quiesce budget. Only a dispatcher still running at +the end of the whole budget reports `dispatcherStopped=false`, exactly as before the cap. +Re-probing the dispatcher is safe where re-reading the register (below) is not: a stopped +dispatcher stays stopped. + +The report is taken from **one read at the deadline**, before the queue is released: + +``` +WARN CommandServiceImpl - Command service drain incomplete - dispatcherStopped=true, + activeCommands=2, inFlight=[cmd-0abc(task-submit), cmd-0def(cluster-create)], timeout=PT20S +``` + +Two reads could describe two different instants, and re-reading after `close()` would be +worse than untidy: `close()` can consume what is left of the budget, so a task that +outlived the deadline but finished during it would make the later read empty and the +caller would be told the drain succeeded. For the same reason the register is **not** +published on the `CommandService` interface — a caller re-reading it after `drain()` +returned would race the very work being reported. `CommandQueueGracefulShutdown` records +the budget that expired and points at this line. + +**What the drain actually costs, measured.** 30 days of production (2026-07-14 → 08-12): +**12 drain events, 10 clean, 2 incomplete.** Every clean drain finished in **13–52 ms** — +the wait loop exits on its first poll with the register already empty. So the drain is not +a rollout-latency cost, and `drain-timeout: 20s` is a ceiling that is essentially never +approached. Both incomplete drains reported `dispatcherStopped=false` with the *dispatcher* +failing to quiesce inside the budget, not a handler — those were pre-#913 pods where +`checkStatus()` ran synchronously on the dispatcher thread. That cause is gone, but the +*shape* is now bounded whatever the cause: with the dispatcher wait capped at a quarter, a +dispatcher that never quiesces costs 5s of the 20s and the register wait still gets 15s, +instead of being skipped entirely. + +**Rejected: an age-based "drainable work" predicate.** Making the wait exit on "no +in-flight invocation younger than N" was implemented and withdrawn. Two reasons, both +worth recording so it is not re-proposed: the measurement above shows there is no wait to +shorten, and `drain-timeout` is documented as sized against the worst observed batch launch +(~7s), so any such cap has to sit above ~10s to avoid abandoning the very work #888 created +the drain to protect — leaving it to save ~5s on the minority of rollouts that have +anything in flight at all. Abandoning an invocation is also not free: a handler that +catches broadly and records a terminal result can settle the command in Redis while its +domain rows stay PENDING. + +### What stays, and one honest delta + +- Single dispatcher thread; `pollInterval` idle backoff; the exponential error + backoff; `check-status-interval` (default 45s) as the poll clock for handler-declared + PROCESSING — transient-error retries and crash recovery stay on the claim cadence. +- The blocking executor, `submitCounted`, the `drain()` wait-then-release shape, + `markProcessing` (single retry + verify) — now reached through + `recordProcessingDeclaration`, which distinguishes the *first* declaration (state still + PENDING: write the transition) from a subsequent poll (state already PROCESSING: + write-free, except to clear a recovered error streak) — `recordError`, the CAS + `update()`. +- Handlers: signatures, semantics, and the existing #890 re-executability + contract. Nothing else is asked of them — the shutdown refusal in §3 is decided at + the routing boundary, so no handler participates in it. +- **Delta (review finding #8)**: `inflight` — and therefore `activeCommands()`, + the readiness indicator, `drain()`'s wait and the admission cap — now covers + `checkStatus` polls too, which previously ran uncounted on the dispatcher + thread. This is an improvement (drain no longer abandons an in-flight poll), + but it inverted one drain test's premise: `CommandServiceDrainTest` was rewritten, + not merely adapted. +- **Since #913**: the bare `inflight` counter became the register described in §5, so + the drain's count and the identities it reports come from one structure; and `drain()` + / `stop()` now raise a shutdown signal that `runCommand` consults before routing + (§3). Both are additive to the model above — no change to ownership, settlement or + the lease. + +## Guarantees + +- **No overlap while the owner lives**: ownership and execution share one + lifetime on one object. A steal requires the idle clock to pass the + visibility timeout, which a live task's heartbeat prevents — regardless of how long + the handler runs (requirement 1, verbatim). +- **Crash recovery**: heartbeats stop, the entry redelivers after one claim + cycle, the state still says PENDING → re-execute. No false SUCCEEDED, no + unpollable PROCESSING, no handler participation. +- **Nearly exactly-once** — the residual duplicate windows: + 1. Crash redelivery re-executing work whose side effects partially landed — + the pre-existing at-least-once contract, inherent. + 2. A renewal outage exceeding `VT − 2·(VT/4)` after the last successful tick: + a steal races a still-live handler. Unfenced by design, but **observable**: + the ownership check detects the loss on the next successful tick, stops + renewing (no ping-pong re-seizure), and counts it (`lease-lost`). The CAS + makes the first terminal result win and the loser's write refuse; external + side effects in the window can double — closable only with cloud-side + idempotency. + 3. Duplicate *entries* for one command (re-drive paths, #906) are out of + scope — deliberately: the normal flow enqueues exactly one entry per + command, and this model removes the re-drive's reason to exist (an entry + is never lost: throw → retry, crash → redeliver, completion → ack). If + evidence ever shows duplicate entries racing live executions, a + command-keyed gate (Proposal B's mechanism) composes onto this design at + the task boundary. + +## Pros + +- One ownership system, zero new keys, zero state-schema change, zero rollout + hazard, no watchdog-over-a-side-lease. (Deploy note: during a rolling window, + old-code replicas don't heartbeat — their in-flight work keeps today's + semantics and stealability; new-code entries are protected. No wire-level + incompatibility: the Redis stream format is unchanged.) +- Deletes more than it adds: `executeWithTimeout`, the 1s budget, abandoned + executions and discarded results all go; completions ack immediately with + their real outcome. +- PROCESSING regains an honest meaning → the crash-recovery handler-contract + problem class (audits, restart signals) is unreachable by construction. +- Head-of-line blocking disappears (the dispatcher never waits on a handler — + today it waits up to 1s on execute and unboundedly on checkStatus). +- Concurrency and shutdown semantics preserved, plus an explicit admission cap + where there was none. + +## Cons + +- The vendored queue consumer API changes (deferred settlement); both + `RedisWorkQueue` and `LocalWorkQueue` implement lease semantics. + Contained: sched is the only consumer. +- Heartbeat rides on Redis health: the quantified outage window above re-opens + overlap (the "nearly") — now with detection and a counter, not silently. +- The renewal scheduler is a new moving part: Throwable-contained tick, batched + per-queue round-trips, age backstop, and its own metrics. +- Per-entry (not per-command) exclusion: duplicate entries from re-drive paths + are out of scope, as argued above. + +## Alternative considered — synchronous execution on N dispatcher threads + +The earlier draft ran handlers synchronously inside the consumer callback and +scaled with N (virtual-thread) dispatchers. Same ownership model, same +guarantees; rejected in favor of task-settled delivery because the latter keeps +today's execution topology (executor, concurrency under an explicit cap, +`drain()` machinery) and keeps the claim path single-threaded, at the cost of +one contained consumer-API change. + +## Essential tests (Redis testcontainer) + +1. **The requirement, verbatim**: a handler sleeping well past a shortened + visibility timeout → a second consumer's `XAUTOCLAIM` finds nothing; exactly one + execution; the entry acks with the handler's result. +2. **Crash recovery**: kill the heartbeat (simulate replica death) mid-execute → + entry redelivered after one claim cycle, state PENDING, re-executed exactly + once elsewhere. +3. **Poll model**: handler returns PROCESSING → lease released, `checkStatus` + re-polled on the check-status interval (claim cadence when at the floor); a slow + `checkStatus` is not stolen mid-poll. A delayed retry holds the lease — + unstealable, never leak-pruned — and redelivers no earlier than the delay. +4. **Renewal margin**: renewal failing for up to two ticks (< VT − 2P total) → + no steal; renewal recovers. Tick duration independent of in-flight count + (batched renewal — one pipelined XPENDING batch + one XCLAIM per queue). + The ownership check stays exact under a large foreign PEL: a stolen leased + entry bracketed by hundreds of other consumers' pending entries is still + detected, dropped and counted — never re-seized. +5. **Ownership check**: an entry force-claimed by another consumer between + ticks → the next tick drops it (lease-lost counter), does NOT re-seize it, + and the thief's execution proceeds unmolested. +6. **Settlement**: ack from the task thread removes the entry (XACK+XDEL); + `retry()` leaves it pending; double-settlement is a no-op; a renewal racing + an ack does not resurrect the entry; a consumer that THROWS settles as RETRY + and leaves nothing registered. +7. **Admission cap**: with `max-concurrency` saturated the dispatcher stops + claiming (saturated counter increments); entries drain as tasks finish; no + unbounded task spawn under a backlog flood. +8. **Retry-on-throw**: a throwing `execute()` → entry redelivered, re-executed; + errorsCount incremented; no PROCESSING fabricated. +9. **Drain**: in-flight tasks (execute AND checkStatus polls) settle their + leases before `drain()` returns; leases of tasks that outlive the budget + stop renewing → redelivered later. The incomplete report is taken at the + deadline, so a task that finishes during `queue.close()` still counts as + outliving the budget — a post-close re-read would flip the outcome to + "drained" and the test asserts it does not. +10. **Age backstop**: a leaked lease (settlement suppressed in test, no live + owner bound) stops being renewed after 3× the visibility timeout and the entry + recovers via the claim cycle; a lease whose bound owner is still alive is + never age-pruned, however old. +11. **Shutdown refusal**: a delivery claimed just before `drain()` — and one + claimed after `stop()` — is not routed to its handler, writes no state, and + settles as retry rather than ack; a *terminal* delivery is still acked while + shutting down (the guard sits after the terminal branch); a restarted service + routes deliveries again. Driven through the real `drain()`/`stop()`/`start()` + entry points, so the tests prove the arming and clearing, not merely the guard. +12. **In-flight register**: names the running command by id and type, forgets it + when the task ends, reports several commands in a stable order, and leaks no + entry when a task throws or the executor rejects the submit. + +## Sequencing — as delivered (#913, branch `command-message-lease`) + +1. `lib-data-workqueue` / `lib-data-workqueue-redis`: `MessageLease` + `Decision` consumer API, + queue-scoped registry, batched renewal with ownership check + age backstop + + metrics, lease semantics in `LocalWorkQueue`, `ready()` admission and + metrics mapping in `AbstractWorkQueue`. +2. `lib-cmd-queue-redis`: `processCommand`/`dispatchCommand`/`runCommand` as + drafted; delete `executeWithTimeout` and the `execute-timeout` config; add + `max-concurrency`; rewrite `CommandServiceDrainTest`; adapt the remaining + tests; add the container suite above. +3. `sched-app`: `SchedCommandConfig` + `application.yml` (drop + `execute-timeout`, add `max-concurrency`, update the model notes). + +## Review log + +Adversarial review (4 lenses × verification, 22 agents): 13 findings confirmed, +5 refuted. Folded in: queue-scoped registry keying (#1); batched per-queue +renewal + VT/4 period + honest margin math + tick-duration warn (#2, #5, #10); +ownership check preventing re-seize ping-pong + lease-lost observability (#3, +#12); exception-safe registration bracket, unregister-first ack, finally-settle +in runCommand, age backstop (#4, #6, #9); Throwable-contained tick + renewal +metrics (#11, #13); `Decision` naming and metrics mapping incl. saturated +counter (#7, #13); drain-delta honesty + `CommandServiceDrainTest` rewrite (#8). + +### After the merge + +- **#963** — the in-flight register replaces the bare counter, and an incomplete drain + names what was running instead of only counting it (§5). Review note folded in: the + report is read once at the deadline rather than rebuilt by the caller afterwards. +- **#964** (merged into #963) — the shutdown refusal in `runCommand` (§3), plus the + contract stated on `CommandService.start()` / `stop()` / `drain()`, since for a vendored + library the interface is what a consumer reads. +- **#965** — closed unmerged. It placed the same "don't start work that cannot finish" + decision at four hand-placed checkpoints *inside* handler business logic, each needing + its own rethrow arm ahead of the generic `catch (Exception)`. A missing arm surfaces as + a task failed with no exit code, and the invariant broke once during the PR's own + development. The refusal at the routing boundary (§3) covers the same window with no + handler participation. +- **Prod measurement** (§5) retired levers that this document's #955 follow-up had + proposed: shortening `drain-timeout`, splitting the in-flight counters, and an + age-based drainable-work predicate. diff --git a/lib-cmd-queue-redis/README.md b/lib-cmd-queue-redis/README.md index 128fdb91..40060c11 100644 --- a/lib-cmd-queue-redis/README.md +++ b/lib-cmd-queue-redis/README.md @@ -1,30 +1,42 @@ # lib-cmd-queue-redis -Asynchronous command queue for executing long-running tasks with persistent state tracking and automatic status polling. +> **1.0.0 replaces the reverted 0.5–0.7 line.** This is the lease-based command queue that +> [#84](https://github.com/seqeralabs/libseqera/pull/84)/[#86](https://github.com/seqeralabs/libseqera/pull/86)/[#87](https://github.com/seqeralabs/libseqera/pull/87)/[#89](https://github.com/seqeralabs/libseqera/pull/89) +> built and [#100](https://github.com/seqeralabs/libseqera/pull/100) withdrew, re-landed on +> top of [`lib-data-workqueue`](../lib-data-workqueue/README.md) 2.0.0 with the hardening the +> scheduler added while running it: a per-command write mutex guarding `CommandState` +> transitions, configurable lock timings and a caller-budgeted `close()`, a retry-safe +> PROCESSING mark (`markProcessing()`), rejection-safe in-flight counting (`submitCounted()`), +> and a wait-once `close()`. +> +> **This is a breaking change over 0.4.0**, whose `CommandServiceImpl` dispatched +> synchronously on `lib-data-stream-redis` with no lease. `CommandQueue` now extends +> `AbstractWorkQueue`; `queueName()` (was `streamName()`) still returns +> `name() + "/v1"`, so Redis keys and the consumer group are untouched. +> +> `CommandState` transitions use the versioned compare-and-swap of +> `lib-data-store-state-redis` 1.2.0+: `CommandState` implements `VersionAware` and the CAS +> witness is a store-stamped version, not re-serialized bytes — byte-equality CAS broke +> cross-replica under JVM-dependent property ordering. +> +> `CommandStatus` follows #86's naming: `SUBMITTED` → `PENDING` and `RUNNING` → `PROCESSING`, +> with `@JsonAlias` on both. **The aliases are load-bearing and must never be removed** — +> `CommandState` is persisted as Jackson-encoded JSON with the status as a bare enum name, so +> state written by the previous naming only deserializes because of them. +> +> **Versions 0.5.0, 0.5.1, 0.6.0 and 0.7.0 remain published but are abandoned** (reverted by +> #100). Do not depend on them; upgrade from 0.4.0 straight to 1.0.0. See `changelog.txt` for +> the downgrade hazard on command state written by 0.7.0. ## Installation -Add this dependency to your `build.gradle`: - ```gradle dependencies { - implementation 'io.seqera:lib-cmd-queue-redis:0.4.0' + implementation 'io.seqera:lib-cmd-queue-redis:1.0.0' } ``` -## Features - -- Fire-and-forget command submission -- Typed parameters and results with JSON serialization -- Status transitions: `SUBMITTED` → `RUNNING` → `SUCCEEDED`/`FAILED`/`CANCELLED` -- Automatic timeout handling for long-running commands -- Periodic status checking for async commands -- Command cancellation support -- Persistent storage using Redis or in-memory backend - -## Usage - -### Define Command Parameters and Result +### Define Command Parameters ```java // Command parameters - must have default constructor for Jackson @@ -78,7 +90,7 @@ public class AsyncProcessingHandler implements CommandHandler execute(Command command) { // Start async job externalService.startJob(command.id(), command.params()); - return CommandResult.running(); // checkStatus() will be called later + return CommandResult.processing(); // checkStatus() will be called later } @Override @@ -86,7 +98,7 @@ public class AsyncProcessingHandler implements CommandHandler state = commandService.getState(commandId); // Get result when complete ProcessingResult result = commandService.getResult(commandId, ProcessingResult.class).orElseThrow(); -// Stop consuming commands (e.g. during shutdown) +// Graceful shutdown: refuse new work, wait (bounded) for in-flight handlers to +// finish while collaborators are still usable, then release the queue. Returns +// false when work was still running at the deadline. activeCommands() reports +// the in-flight count for readiness probes. +commandService.drain(Duration.ofSeconds(20)); + +// Immediate variant — releases the queue without waiting for in-flight handlers commandService.stop(); ``` ## Metrics (optional) Since `0.4.0`, `CommandQueue` exposes a second constructor that forwards an optional -[`StreamMetrics`](https://github.com/seqeralabs/libseqera/tree/master/lib-data-stream-redis) -handle to the underlying `AbstractMessageStream`. Subclasses that want to publish -Micrometer metrics construct a `MicrometerStreamMetrics` from a `MeterRegistry` and pass -it through: +[`QueueMetrics`](../lib-data-workqueue/README.md#metrics-optional) handle to the underlying +`AbstractWorkQueue`. Subclasses that want to publish Micrometer metrics construct a +`MicrometerQueueMetrics` from a `MeterRegistry` and pass it through: ```java import io.micrometer.core.instrument.MeterRegistry; import io.micronaut.core.annotation.Nullable; -import io.seqera.data.stream.metrics.MicrometerStreamMetrics; +import io.seqera.data.workqueue.WorkQueue; +import io.seqera.data.workqueue.metrics.MicrometerQueueMetrics; public class MyCommandQueue extends CommandQueue { @Inject - public MyCommandQueue(MessageStream target, @Nullable MeterRegistry registry) { + public MyCommandQueue(WorkQueue target, @Nullable MeterRegistry registry) { super(target, registry != null - ? new MicrometerStreamMetrics(registry, "my-cmd-queue") + ? new MicrometerQueueMetrics(registry, "my-cmd-queue") : null); } @@ -146,24 +164,38 @@ public class MyCommandQueue extends CommandQueue { ``` The 1-arg constructor is unchanged: existing subclasses continue to compile and run -with no metrics. See [`lib-data-stream-redis`](../lib-data-stream-redis/README.md) for the -list of published meters (`seqera.stream.entries`, `seqera.stream.messages`, -`seqera.stream.processing`) and their tags. +with no metrics. See [`lib-data-workqueue`](../lib-data-workqueue/README.md) for the +list of published meters (`seqera.workqueue.entries`, `seqera.workqueue.messages`, +`seqera.workqueue.processing`) and their tags. ## Command Status Flow ``` -submit() ──▶ SUBMITTED ──pickup──▶ RUNNING ─┬─success──▶ SUCCEEDED - ├─error────▶ FAILED - └─cancel───▶ CANCELLED +submit() ──▶ PENDING ──pickup──▶ PROCESSING ─┬─success──▶ SUCCEEDED + ├─error────▶ FAILED + └─cancel───▶ CANCELLED ``` +A handler that **throws** is not terminally failed: the message stays queued and is +redelivered, with the consecutive-error streak tracked on the state (`errorsCount`, +`error`). A *permanent* failure is signalled by returning a FAILED `CommandResult` — +deciding that is the domain layer's job, never the queue's (see seqeralabs/sched#712, seqeralabs/sched#890). + ## Testing ```bash ./gradlew :lib-cmd-queue-redis:test ``` +## Design notes + +- [`docs/plans/command-execution-guarantee-message-lease.md`](../docs/plans/command-execution-guarantee-message-lease.md) + — the execution guarantee the message lease provides, and why the lease is settled by the + handler rather than the dispatcher. +- [`docs/plans/2026-07-31-command-state-write-mutex-design.md`](../docs/plans/2026-07-31-command-state-write-mutex-design.md) + — the per-command write mutex guarding `CommandState` transitions, and the alternatives + rejected on the way to it. + ## License Apache License 2.0 diff --git a/lib-cmd-queue-redis/VERSION b/lib-cmd-queue-redis/VERSION index 1d0ba9ea..3eefcb9d 100644 --- a/lib-cmd-queue-redis/VERSION +++ b/lib-cmd-queue-redis/VERSION @@ -1 +1 @@ -0.4.0 +1.0.0 diff --git a/lib-cmd-queue-redis/build.gradle b/lib-cmd-queue-redis/build.gradle index 828cdff2..4f2c0987 100644 --- a/lib-cmd-queue-redis/build.gradle +++ b/lib-cmd-queue-redis/build.gradle @@ -32,12 +32,16 @@ dependencies { // JSON serialization implementation project(':lib-serde-jackson') - // Message stream - implementation project(':lib-data-stream-redis') + // The work queue this command queue is built on. `api`, not `implementation`: + // CommandQueue publicly extends AbstractWorkQueue and callers construct a + // RedisWorkQueue / LocalWorkQueue to hand to it, so those types are part of + // this module's surface. + api project(':lib-data-workqueue') implementation project(':lib-serde-moshi') - // State store - implementation project(':lib-data-store-state-redis') + // State store. `api`, not `implementation`: CommandState publicly implements + // VersionAware, so the interface is part of this module's surface. + api project(':lib-data-store-state-redis') // Utilities (includes type resolution) implementation project(':lib-lang') @@ -50,6 +54,14 @@ dependencies { // Test dependencies testImplementation testFixtures(project(':lib-fixtures-redis')) + // The Redis backend is a test-only dependency: RedisWorkQueue replicas are built by + // hand in the lease container tests (CommandServiceLeaseRedisTest) + testImplementation project(':lib-data-workqueue-redis') + // JedisPool is wired directly into those manually-built RedisWorkQueue replicas; the + // Groovy compiler also resolves the full CommandQueue supertype hierarchy, which + // reaches lib-retry (an `implementation` dep of lib-data-workqueue, so not transitive) + testImplementation 'redis.clients:jedis:5.1.4' + testImplementation project(':lib-retry') testImplementation "io.micronaut:micronaut-inject-groovy:${micronautCoreVersion}" testImplementation "io.micronaut.test:micronaut-test-spock:${micronautTestVersion}" testImplementation 'org.apache.groovy:groovy' diff --git a/lib-cmd-queue-redis/changelog.txt b/lib-cmd-queue-redis/changelog.txt index d90fbfca..721b8ce4 100644 --- a/lib-cmd-queue-redis/changelog.txt +++ b/lib-cmd-queue-redis/changelog.txt @@ -1,5 +1,39 @@ # lib-cmd-queue-redis changelog +1.0.0 - 19 Aug 2026 +- BREAKING change over 0.4.0. Re-lands the lease-based command queue withdrawn by PR #100, + promoted from the Seqera scheduler where it has been running, and supersedes the abandoned + 0.5.0/0.5.1/0.6.0/0.7.0 line for good. +- Depends on lib-data-workqueue + lib-data-workqueue-redis 2.0.0 instead of + lib-data-stream-redis. CommandQueue extends AbstractWorkQueue; queueName() (was + streamName()) still returns name() + "/v1", so the Redis keys and the consumer group are + unchanged. +- CommandServiceImpl replaces 0.4.0's synchronous dispatch with lease-held execution: the + handler settles the message, a command that is not yet terminal holds its lease and is + re-invoked in-process, and a handler that throws is retried rather than terminal-failed. +- CommandStatus: SUBMITTED -> PENDING, RUNNING -> PROCESSING, both carrying @JsonAlias. The + aliases are load-bearing - CommandState is persisted as JSON with the status as a bare enum + name - and must never be removed. +- CommandState transitions use the versioned compare-and-swap of lib-data-store-state-redis + 1.2.0+: CommandState implements VersionAware and the CAS witness is a store-stamped version, + not re-serialized bytes. Byte-equality CAS broke cross-replica under JVM-dependent property + ordering. +- Hardening added while the scheduler ran this code: a per-command write mutex guarding + CommandState transitions, configurable lock timings, a caller-budgeted close(), a retry-safe + PROCESSING mark (markProcessing()), rejection-safe in-flight counting (submitCounted()), and + a wait-once close(). +- Error tracking on CommandState (errorsCount, error, modifiedAt), re-landed from the reverted + 0.6.0. +- api rather than implementation for lib-data-workqueue and lib-data-store-state-redis: + CommandQueue publicly extends AbstractWorkQueue and CommandState publicly implements + VersionAware, so both are part of this module's surface. +- DOWNGRADE WARNING: this release persists command state with the PENDING/PROCESSING names and + a 7-day TTL. 0.4.0 cannot decode them (PROCESSING fails deserialization; PENDING was a dead + constant there), so if 1.0.0 has run against a given Redis, wait out the TTL or flush the + affected command-state keys before rolling back onto 0.4.0. +- Design notes: docs/plans/command-execution-guarantee-message-lease.md and + docs/plans/2026-07-31-command-state-write-mutex-design.md. + REVERTED - 0.5.0, 0.5.1, 0.6.0, 0.7.0 (13 - 18 Jul 2026) - These releases have been reverted; this module is back at 0.4.0 and to its lib-data-stream-redis dependency: diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandConfig.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandConfig.java index 6d5c07e7..3db5c6b6 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandConfig.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandConfig.java @@ -37,12 +37,20 @@ default Duration pollInterval() { } /** - * Timeout for synchronous command execution. - * If execute() takes longer than this, the command is marked as RUNNING - * and checkStatus() will be called on subsequent queue deliveries. + * Admission cap: the maximum number of handler executions in flight at once. + * The dispatcher stops claiming messages from the queue while the cap is + * reached, resuming as tasks finish. + * + *

Replaces the implicit throttle the deleted 1-second execute-timeout used + * to be: with fire-and-submit dispatch every delivery spawns a task immediately, + * so without an explicit cap a backlog flood would spawn unbounded tasks. + * + *

Size it below the shared JDBC connection pool minus request-path headroom: + * handler tasks make several sequential JDBC round-trips each, and bursts beyond + * the cap should queue in the work queue, not in the connection pool. */ - default Duration executeTimeout() { - return Duration.ofSeconds(1); + default int maxConcurrency() { + return 20; } /** @@ -52,4 +60,30 @@ default Duration executeTimeout() { default Duration stateTtl() { return Duration.ofDays(7); } + + /** + * Bound on the compare-and-swap retry loop of a command state transition. A miss + * means another writer transitioned the state between the read and the write; the + * loop re-reads and re-applies, converging in one or two rounds under real + * contention (at most a handful of writers ever touch one command). The bound is a + * livelock backstop, not a throughput tunable — must be positive. + */ + default int stateUpdateAttempts() { + return 5; + } + + /** + * Re-poll cadence for commands whose handler declared PROCESSING: how often + * {@code checkStatus()} is invoked while the async work is in flight. + * + *

Deliberately decoupled from the queue's visibility timeout, which paces crash + * detection and transient-error retries: shortening that clock must not multiply + * the polling load {@code checkStatus()} puts on its downstream dependencies + * (database reads, cloud describe calls). A cadence at or below the visibility + * timeout degrades to the claim cadence — the visibility timeout is the effective + * floor. + */ + default Duration checkStatusInterval() { + return Duration.ofSeconds(45); + } } diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandHandler.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandHandler.java index 568704fb..c62fc836 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandHandler.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandHandler.java @@ -34,9 +34,9 @@ public interface CommandHandler { /** * Execute the command and return a result. * This method is executed asynchronously via an executor service. - * If execution takes longer than 1 second, the command is marked as RUNNING and + * If execution takes longer than 1 second, the command is marked as PROCESSING and * {@link #checkStatus} will be called periodically to check completion. - * For long-running commands, return {@link CommandResult#running()} to indicate + * For long-running commands, return {@link CommandResult#processing()} to indicate * the operation is in progress. * * @param command The command to execute @@ -46,15 +46,15 @@ public interface CommandHandler { /** * Check the status of a long-running command. - * Called periodically for commands in RUNNING state until a terminal status is returned. + * Called periodically for commands in PROCESSING state until a terminal status is returned. * The command parameter provides typed access to params via {@code command.params()}. * The state parameter provides access to timing and status information. * * @param command The command being checked (provides typed params access) * @param state The current command state (timing, status info) - * @return The result indicating current status (RUNNING to continue, or terminal status) + * @return The result indicating current status (PROCESSING to continue, or terminal status) */ default CommandResult checkStatus(Command

command, CommandState state) { - return CommandResult.running(); + return CommandResult.processing(); } } diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandQueue.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandQueue.java index 9544d2ef..f5ef97d1 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandQueue.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandQueue.java @@ -17,11 +17,11 @@ package io.seqera.data.command; import io.micronaut.core.annotation.Nullable; -import io.seqera.data.stream.AbstractMessageStream; -import io.seqera.data.stream.MessageConsumer; -import io.seqera.data.stream.MessageStream; -import io.seqera.data.stream.metrics.NoopStreamMetrics; -import io.seqera.data.stream.metrics.StreamMetrics; +import io.seqera.data.workqueue.AbstractWorkQueue; +import io.seqera.data.workqueue.MessageConsumer; +import io.seqera.data.workqueue.WorkQueue; +import io.seqera.data.workqueue.metrics.NoopQueueMetrics; +import io.seqera.data.workqueue.metrics.QueueMetrics; import io.seqera.serde.encode.StringEncodingStrategy; import io.seqera.serde.moshi.MoshiEncodeStrategy; import jakarta.annotation.PreDestroy; @@ -30,16 +30,16 @@ /** * Abstract message queue for command processing. - * Extends AbstractMessageStream to provide async, fire-and-forget command submission. + * Extends AbstractWorkQueue to provide async, fire-and-forget command submission. * * Subclasses must implement {@link #name()} and {@link #pollInterval()} * to configure the queue behavior. */ -public abstract class CommandQueue extends AbstractMessageStream { +public abstract class CommandQueue extends AbstractWorkQueue { private static final Logger log = LoggerFactory.getLogger(CommandQueue.class); - public CommandQueue(MessageStream target) { + public CommandQueue(WorkQueue target) { super(target); log.info("Created command queue - name={}", name()); } @@ -47,13 +47,13 @@ public CommandQueue(MessageStream target) { /** * Constructs a command queue with optional metrics instrumentation. * - * @param target the underlying {@link MessageStream} - * @param metrics the {@link StreamMetrics} to publish to, or {@code null} for no-op + * @param target the underlying {@link WorkQueue} + * @param metrics the {@link QueueMetrics} to publish to, or {@code null} for no-op */ - public CommandQueue(MessageStream target, @Nullable StreamMetrics metrics) { + public CommandQueue(WorkQueue target, @Nullable QueueMetrics metrics) { super(target, metrics); log.info("Created command queue - name={}; metrics={}", - name(), metrics != null && !(metrics instanceof NoopStreamMetrics) ? "enabled" : "disabled"); + name(), metrics != null && !(metrics instanceof NoopQueueMetrics) ? "enabled" : "disabled"); } @Override @@ -62,15 +62,15 @@ protected StringEncodingStrategy createEncodingStrategy() { } /** - * The name of the command queue. Used for logging and stream name derivation. + * The name of the command queue. Used for logging and queue name derivation. */ @Override protected abstract String name(); /** - * The name of the message stream, derived from {@link #name()}. + * The name of the underlying work queue, derived from {@link #name()}. */ - protected String streamName() { + protected String queueName() { return name() + "/v1"; } @@ -80,16 +80,20 @@ protected String streamName() { * @param msg the command message to queue */ public void submit(CommandMsg msg) { - offer(streamName(), msg); + offer(queueName(), msg); } /** - * Register a consumer for commands. + * Register a consumer for commands. The consumer returns a + * {@link MessageConsumer.Decision} per delivery — {@code ACK} to settle, {@code RETRY} + * to redeliver after the visibility timeout, {@code DEFERRED} to transfer settlement to a + * task via the {@link io.seqera.data.workqueue.MessageLease} — and may gate admission + * through {@link MessageConsumer#ready()}. * * @param consumer the consumer to process commands */ public void addConsumer(MessageConsumer consumer) { - addConsumer(streamName(), consumer); + addConsumer(queueName(), consumer); } /** @@ -98,7 +102,7 @@ public void addConsumer(MessageConsumer consumer) { * @return number of pending commands */ public int length() { - return length(streamName()); + return length(queueName()); } @PreDestroy diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandResult.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandResult.java index 5f85da2e..6dbf6df8 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandResult.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandResult.java @@ -45,10 +45,10 @@ public static CommandResult failure(String error) { } /** - * Indicate that the command is still running (for long-running commands). + * Indicate that the command is still being processed (for long-running commands). */ - public static CommandResult running() { - return new CommandResult<>(CommandStatus.RUNNING, null, null); + public static CommandResult processing() { + return new CommandResult<>(CommandStatus.PROCESSING, null, null); } /** diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandService.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandService.java index 84ba84e9..b4117cc1 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandService.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandService.java @@ -16,6 +16,7 @@ */ package io.seqera.data.command; +import java.time.Duration; import java.util.Optional; /** @@ -91,12 +92,51 @@ public interface CommandService { * Start consuming commands from the queue. * Must be called AFTER all handlers are registered to avoid race conditions * where messages are processed before handlers are available. + * + *

Also clears the shutdown signal raised by {@link #stop()} or {@link #drain(Duration)}, so a + * service started again routes deliveries to their handlers instead of refusing them. */ void start(); /** * Stop consuming commands from the queue. * Called during shutdown to gracefully stop processing. + * + *

This releases the queue immediately and does not wait for handler executions already in + * progress. Prefer {@link #drain(Duration)} when those executions depend on resources — a + * database connection pool, for instance — that are about to be torn down. + * + *

It also raises the shutdown signal: a delivery claimed just before this call is settled for + * redelivery rather than routed to its handler. Only work not yet started is declined — an + * execution already under way is unaffected. */ void stop(); + + /** + * Stop claiming new commands and wait for the ones already being handled to finish. + * + *

Intended to be called while the rest of the application is still alive, so that a handler + * mid-execution can complete its work and record its outcome instead of failing against + * resources that have already been closed. On return the queue is released, as with + * {@link #stop()}. + * + *

Raises the shutdown signal on entry, with the same effect as {@link #stop()}: a delivery + * claimed just before the call is settled for redelivery rather than routed, so the wait below is + * not extended by work begun after it started. + * + *

Deliberately framework-agnostic: the caller decides what triggers it and how the timeout + * relates to any container-level shutdown budget. + * + * @param timeout + * how long to wait for in-flight commands to finish + * @return + * {@code true} if nothing was left running, {@code false} if the timeout was reached with + * commands still in flight — the caller may then log, report, or proceed regardless + */ + boolean drain(Duration timeout); + + /** + * @return the number of command handler executions currently in progress + */ + int activeCommands(); } diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandServiceImpl.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandServiceImpl.java index 698b1e11..f2385b0b 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandServiceImpl.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandServiceImpl.java @@ -16,16 +16,21 @@ */ package io.seqera.data.command; +import java.time.Duration; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; import io.micronaut.scheduling.TaskExecutors; import io.seqera.data.command.store.CommandStateStore; +import io.seqera.data.workqueue.MessageConsumer; +import io.seqera.data.workqueue.MessageConsumer.Decision; +import io.seqera.data.workqueue.MessageLease; import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.inject.Singleton; @@ -36,20 +41,36 @@ * Implementation of the command service. * Handles queue consumption and command execution with proper multi-replica support. * - *

Processing flow: + *

Processing flow (task-settled delivery — every handler invocation runs on the + * blocking executor and settles its own message lease when it finishes): *

    - *
  • If command is already RUNNING → call checkStatus() synchronously
    • - *
  • If command is not RUNNING → execute asynchronously with 1-second timeout: - *
      - *
    • If completes within timeout → process result immediately
      • - *
    • If times out → mark as RUNNING, retry later via queue
      • - *
    - *
    • - *
  • If result is RUNNING → return false (message stays in queue for retry)
    • - *
  • If result is terminal → return true (message removed from queue)
    • - *
  • If the handler throws → return false so the message is retried; a throw is treated as - * transient, never as a terminal failure (deciding permanent failure is the domain - * layer's job, see seqeralabs/sched#712)
    • + *
  • Stale delivery (state missing or already terminal) → settle {@code ACK}
    • + *
  • No handler registered → record FAILED, settle {@code ACK} (or {@code RETRY} + * when the write refuses)
    • + *
  • Otherwise → submit the handler invocation to the blocking executor and return + * {@code DEFERRED}: the task owns the lease, which stays renewed for as long as + * the task runs, so a slow handler's entry never goes stalled mid-flight
    • + *
  • The task routes explicitly on the state: PROCESSING → checkStatus(), + * PENDING → execute(), a terminal snapshot → stale, acked. PROCESSING only + * ever means "the handler returned PROCESSING" — never the queue's impatience
    • + *
  • ...unless a shutdown was signalled ({@code drain()} or {@code stop()}) before the task + * reached that routing, in which case it does not route at all: nothing has been mutated, so + * the delivery is settled as a retry rather than starting work that is not guaranteed to + * finish inside the shutdown budget
    • + *
  • Handler result PROCESSING → record PROCESSING (first time only), settle the lease as + * a delayed retry: the next checkStatus() poll runs on {@code checkStatusInterval}, + * decoupled from the visibility timeout (which paces crash detection and error + * retries)
    • + *
  • Terminal result → record it, settle the lease as ack; a refused write means a + * cancel won underneath — settle as retry, the redelivery acks on the terminal + * check
    • + *
  • If the handler throws → settle as retry; the state is unchanged, so the + * redelivery re-executes a PENDING command or re-polls a PROCESSING one. A throw + * is treated as transient, never as a terminal failure (deciding permanent + * failure is the domain layer's job, see seqeralabs/sched#712)
    • + *
  • Admission cap: the dispatcher stops claiming while + * {@code maxConcurrency} handler executions are in flight + * ({@link MessageConsumer#ready()})
    • *
*/ @Singleton @@ -74,19 +95,92 @@ public class CommandServiceImpl implements CommandService { private volatile boolean started = false; + /** + * Whether this service is shutting down: set by {@link #drain(Duration)} and {@link #stop()}, + * cleared by {@link #start()} so a restarted service accepts work again. + * + *

Distinct from {@code !started}, which only says the dispatcher stopped claiming. This says + * the shutdown is under way, and is what a handler task consults to decide whether starting its + * work is still worth doing. + */ + private volatile boolean draining = false; + + /** + * Handler invocations submitted to {@link #executor} that have not returned yet — both + * execute() calls and checkStatus() polls, which run as tasks alike — each naming the command + * it is running. + * + *

One structure on purpose: it is both the count a drain waits on and the identities a + * shutdown reports. A separate counter alongside it would mean two things to keep in step at + * three mutation sites, and a window where the count and the names disagree — which is the one + * thing a shutdown report must not do. + * + *

Deliberately not derived from {@link #executor}: that pool is shared and + * container-managed, so its queue says nothing about this service. It backs + * {@link #activeCommands()}, the {@link #drain(Duration)} wait, the admission cap + * ({@code ready()}) that stops the dispatcher from claiming while + * {@code config.maxConcurrency()} tasks are in flight, and the in-flight report. + * + *

Keyed by {@link #inflightSeq} rather than by command id, which matters more than it looks: + * two invocations of one command id in flight at once (not expected under the message lease, + * but not structurally impossible) must not collapse into a single entry — that would + * under-count the admission cap and the drain, not merely lose a name. + * + *

The one thing given up is exactness: {@code size()} on a concurrent map is an estimate, + * not a linearizable count. It costs nothing where it is read. The admission decision is + * already a heuristic at an instant — work starts and finishes around it — and the only thread + * that inserts is the dispatcher that reads it, so it always observes its own insert; the only + * staleness comes from other threads' removals, which makes the count read HIGH and the cap + * admit fewer rather than more. The drain's wait converges as soon as the last task removes its + * entry, and is bounded by a deadline regardless. + */ + private final Map inflight = new ConcurrentHashMap<>(); + + /** + * Source of the {@link #inflight} keys: monotonic, never reused for the life of the + * JVM, so a key cannot be recycled onto a later invocation while an earlier one still holds it. + */ + private final AtomicLong inflightSeq = new AtomicLong(); + + /** + * Granularity at which {@link #drain(Duration)} re-checks {@link #inflight}. + */ + private static final long DRAIN_POLL_MILLIS = 50; + @Override public void start() { + // Cleared before the guard, mirroring how stop()/drain() set it before theirs: starting is + // the one operation that means "accept work", so it clears the shutdown signal + // unconditionally. A service started again after a stop()/drain() must route deliveries to + // handlers, not keep refusing them for the life of the JVM. + draining = false; if (started) { log.debug("Command service already started"); return; } started = true; - queue.addConsumer(this::processCommand); + queue.addConsumer(new MessageConsumer<>() { + @Override + public Decision accept(CommandMsg msg, MessageLease lease) { + return processCommand(msg, lease); + } + + @Override + public boolean ready() { + // admission cap: stop claiming while maxConcurrency tasks are in flight + return inflight.size() < config.maxConcurrency(); + } + }); log.info("Command service started - consuming commands"); } @Override public void stop() { + // Signalled before the started guard, so the signal does not depend on stop() having + // anything left to do. In the shipped wiring that is symmetry rather than a fix — every path + // that clears `started` sets this first — but it keeps the invariant local to each entry + // point instead of resting on the order of an earlier call. + draining = true; if (!started) { return; } @@ -96,14 +190,122 @@ public void stop() { } @Override - public

String submit(Command

command) { - // Create submitted state with params object directly (serialized via @JsonTypeInfo) - final var state = CommandState.submitted(command.id(), command.type(), command.params()); + public boolean drain(Duration timeout) { + // Signal the shutdown first, before anything is awaited: from here on a handler task that + // has not started its work refuses to start it (see runCommand), which is what lets the + // waits below finish in the time they were given instead of outlasting them. + draining = true; + if (!started) { + return activeCommands() == 0; + } + started = false; + final long deadline = System.currentTimeMillis() + Math.max(0, timeout.toMillis()); + + // 1. Stop claiming new commands and let the dispatcher finish the message it holds. This + // does not release the queue, so an in-progress handler can still acknowledge. Bounded + // by a fraction of the budget rather than all of it, so step 2 always gets a slice — + // see quiesceBudget(). + final boolean quiesced = queue.awaitQuiescent(quiesceBudget(timeout)); + + // 2. Wait for handler tasks still running on the executor — execute() calls and + // checkStatus() polls alike. These are the ones that matter: they are mid-flight + // against the database, and letting them finish here (and settle their leases) is + // the whole point of draining before the context tears down. + while (!inflight.isEmpty() && System.currentTimeMillis() < deadline) { + try { + Thread.sleep(Math.min(DRAIN_POLL_MILLIS, Math.max(1, deadline - System.currentTimeMillis()))); + } + catch (InterruptedException e) { + log.info("Command service drain interrupted - giving up the in-flight wait early", e); + Thread.currentThread().interrupt(); + break; + } + } + + // ONE read, taken HERE — at the deadline, before the queue is released below — and used for + // everything after it: the outcome, the count and the names. Two reads could describe two + // different instants, and the one that matters is this one. Reading again after close() would + // be worse than untidy: close() can take what is left of the budget, so a task that outlived + // the deadline and finished during it would make the later read empty, and the caller would be + // told the drain succeeded. Nor can the caller read this for itself afterwards — same race, + // one stack frame further out. + final List remaining = activeCommandDetails(); + // 3. Release the queue. Done last so steps 1-2 ran with every collaborator still usable, + // and bounded by what is left of OUR budget: close() would otherwise start a second, + // independent timer of its own, so a dispatcher that never quiesced could push this + // method well past `timeout`. Overrunning matters because the caller's budget is + // typically a container's graceful-shutdown grace period, and exceeding it means being + // hard-stopped mid-drain — the opposite of what draining is for. + queue.close(Duration.ofMillis(Math.max(0, deadline - System.currentTimeMillis()))); + + // Re-probe AFTER close(): step 1's wait is capped at a fraction of the budget, but close() + // just waited again with the leftover, so a dispatcher that outlived its quiesce budget may + // have stopped by now — slow, not stuck, with nothing lost. Unlike the in-flight register + // above, re-reading this cannot lie: a stopped dispatcher stays stopped, and awaitQuiescent + // with a zero budget is a state probe, not another wait. + final boolean stopped = quiesced || queue.awaitQuiescent(Duration.ZERO); + if (!quiesced && stopped) { + log.warn("Command dispatcher exceeded its quiesce budget ({}) but stopped within the drain budget ({})", quiesceBudget(timeout), timeout); + } + if (!stopped || !remaining.isEmpty()) { + // Report WHAT was still running, not just how much. A drain that expires with work in + // flight is an expected shape — a control-plane call can be given a budget longer than + // this whole drain — and only the ids and types tell that apart from something + // unexpected. Both values were read at the deadline, so they describe one instant. + log.warn("Command service drain incomplete - dispatcherStopped={}, activeCommands={}, inFlight={}, timeout={}", + stopped, remaining.size(), remaining, timeout); + return false; + } + log.info("Command service drained - no command left in flight"); + return true; + } + + /** + * How much of the drain budget step 1 may spend waiting for the dispatcher to quiesce. + * + *

Bounded so step 2 always gets a slice. Handed the whole budget — as it was — a dispatcher + * that never quiesces consumes all of it, and because the deadline is taken before step 1 the + * in-flight wait then runs zero iterations: handler tasks mid-flight against + * the database get no grace at all, which is the one thing the drain exists to provide (#888). + * That is the signature of both incomplete drains observed in production, each reporting + * {@code dispatcherStopped=false} (#955). + * + *

A quarter, derived from the caller's budget the way the lease renewal period derives from + * the visibility timeout, so re-tuning {@code drain-timeout} scales both halves together and + * there is no second dial to keep in step with the first. + * + *

Generous by construction rather than by guess: the dispatcher's own work between two + * loop-head checks is a {@code findById} and a dispatch decision — every handler invocation + * runs on the executor, and since #964 a delivery claimed after the shutdown began is not + * routed at all. Whole drains measure 13-52ms in production, so a quarter of a 20s budget is + * two orders of magnitude of headroom for a step that should never need seconds. + */ + private static Duration quiesceBudget(Duration timeout) { + return timeout.dividedBy(4); + } - // Persist to storage and submit to queue + @Override + public int activeCommands() { + return inflight.size(); + } + + /** + * Name the invocations currently in flight, one entry per invocation, as + * {@code ()}. Deliberately NOT on {@link CommandService}: the only consumer is + * this class's own shutdown report, and publishing it would invite a caller to re-read it after + * {@link #drain(Duration)} has returned — which races the very work being reported. + */ + List activeCommandDetails() { + // Sorted so repeated reads — and a log line read by a human — have a stable order; + // the list is bounded by config.maxConcurrency(), so the sort is free. + return inflight.values().stream().sorted().toList(); + } + + @Override + public

String submit(Command

command) { + final var state = CommandState.create(command.id(), command.type(), command.params()); store.save(state); queue.submit(CommandMsg.of(command.id(), command.type())); - log.debug("Command submitted: id={}, type={}", command.id(), command.type()); return command.id(); } @@ -123,16 +325,13 @@ public Optional getResult(String commandId, Class resultType) { @Override public boolean cancel(String commandId) { - final var state = store.findById(commandId).orElse(null); - if (state == null) { + // The CAS update owns the guards: a missing or already-terminal command refuses the + // write. Reporting false is the point — a blind write could discard a result that had + // already been recorded, leaving the caller believing a cancel took effect. + if (!store.update(commandId, CommandState::cancelled)) { + log.info("Command cancel did not take - id={}", commandId); return false; } - - if (state.status().isTerminal()) { - return false; - } - - store.save(state.cancelled()); log.info("Command cancelled: id={}", commandId); return true; } @@ -174,177 +373,251 @@ public P params() { } /** - * Process a command message received from the queue. - * - *

This is the entry point for queue message consumption. It performs initial - * validation and state lookup, then delegates to the type-safe handler method. - * - *

Return value semantics (controls queue behavior): - *

    - *
  • {@code true} = command fully processed, remove message from queue
    • - *
  • {@code false} = command needs retry, keep message in queue for redelivery
    • - *
- * - * @param msg The command message containing commandId and type - * @return true to acknowledge (remove from queue), false to retry later + * Queue consumer entry point. Deliveries settle three ways: ACK for stale/terminal + * messages, RETRY for transient refusals (redelivered after the visibility timeout), + * DEFERRED when a handler task takes the entry lease and settles it on completion. + * A throw out of this method is settled as RETRY by the queue layer. */ - private boolean processCommand(CommandMsg msg) { - // Step 1: Load command state from persistent storage - var state = store.findById(msg.commandId()).orElse(null); + private MessageConsumer.Decision processCommand(CommandMsg msg, MessageLease lease) { + final var state = store.findById(msg.commandId()).orElse(null); if (state == null) { log.error("Command state not found - this should not happen: id={}", msg.commandId()); - return true; + return Decision.ACK; } - - // Step 2: Check if command already reached a terminal state (SUCCEEDED/FAILED/CANCELLED) - // This can happen if another replica processed it, or if it was cancelled. - // Return true to remove the now-stale message from the queue. if (state.status().isTerminal()) { - return true; + return Decision.ACK; } - - // Step 3: Look up the registered handler for this command type - // If no handler is registered, mark as FAILED and remove from queue. final var registration = getHandler(state.type()); if (registration == null) { log.error("No handler for command type: {}", state.type()); - store.save(state.failed("No handler for type: " + state.type())); - return true; + return store.update(state.id(), s0 -> s0.failed("No handler for type: " + s0.type())) + ? Decision.ACK + : Decision.RETRY; } - - // Step 4: Delegate to the type-capturing helper method - // This pattern allows Java to infer concrete type parameters (P, R) from the - // CommandRegistration, enabling type-safe handler invocation without raw types. - return processCommandWithHandler(msg, state, registration); + return dispatchCommand(state, registration, lease); } /** - * Execute command processing with a specific handler registration. - * - *

This helper method captures the type parameters {@code } from the - * {@link CommandRegistration}, allowing type-safe interaction with the handler. - * - *

Processing flow: - *

    - *
  1. If command is already RUNNING → call {@code checkStatus()} to poll for completion
    2. - *
  2. If command is not yet RUNNING → call {@code execute()} with timeout: - *
      - *
    • If completes within timeout → process the result immediately
      • - *
    • If times out → mark as RUNNING, return false to retry later
      • - *
    - *
    3. - *
  3. If result status is RUNNING → return false (keep in queue for polling)
    4. - *
  4. If result status is terminal → update state and return true (done)
    5. - *
- * - * @param msg The original queue message (for logging) - * @param state The current command state from storage (wildcard types) - * @param registration The handler registration with type parameters captured - * @param

The command parameter type - * @param The command result type - * @return true to acknowledge (remove from queue), false to retry later + * Hand the delivery to a handler task. From a successful submit onward the TASK owns + * the lease — runCommand() settles it on every exit path. A rejected submit means + * nothing runs and nothing owns the lease: RETRY, the claim cycle re-delivers. */ - private boolean processCommandWithHandler( - CommandMsg msg, + private MessageConsumer.Decision dispatchCommand( CommandState state, - CommandRegistration registration) { + CommandRegistration registration, + MessageLease lease) { - // Reconstruct the typed Command object from persisted state - // Uses Class.cast() internally for type-safe conversion - final Command

command = toCommand(state, registration); + final Command

command = toCommand(state, registration); // a throw here → RETRY via the queue layer final CommandHandler handler = registration.handler(); + try { + final Future task = submitCounted(() -> runCommand(command, state, handler, lease), describe(state)); + // The task is the lease's owner: bind its liveness so the queue's lease-age + // backstop never mistakes a long-running handler for a registry leak — the + // backstop only prunes when the owning task is provably gone. + lease.bindLiveness(() -> !task.isDone()); + } + catch (RuntimeException e) { + log.error("Command dispatch rejected, will retry: id={}", state.id(), e); + return Decision.RETRY; + } + return Decision.DEFERRED; + } + /** + * Runs on the blocking executor; the entry lease is renewed for as long as this + * takes, so a slow handler's entry can never go stalled mid-flight. The finally guarantees + * every exit — including an Error out of handler code — settles the lease; the + * idempotent settle makes the happy-path ack and the finally's retry compose. + */ + private void runCommand(Command

command, CommandState state, + CommandHandler handler, MessageLease lease) { try { - CommandResult result; - - // Branch based on current command status - if (state.status() == CommandStatus.RUNNING) { - // Command was previously marked as RUNNING (long-running async operation) - // Call checkStatus() to poll the external system for completion - result = handler.checkStatus(command, state); - } else { - // Command not yet running (status is SUBMITTED) - // Execute with timeout to avoid blocking the queue processor indefinitely - result = executeWithTimeout(handler, command); - - // Timeout case: execute() is still running in background thread - // Mark state as RUNNING so next delivery will call checkStatus() instead - if (result == null) { - store.save(state.started()); - return false; // Keep in queue - will retry and call checkStatus() - } + // Terminal snapshot: unreachable via processCommand()'s terminal pre-check, + // but this method stays total rather than trusting its caller — the message + // is stale: ack. Handled BEFORE the handler routing, on its own branch, so + // no handler-result sentinel can ever be confused with it. + if (state.status().isTerminal()) { + lease.ack(); + return; } - - // Handler returned a result - check if command is still in progress - if (result.status() == CommandStatus.RUNNING) { - // Handler explicitly returned RUNNING (e.g., async job not yet complete) - // Ensure state reflects RUNNING status for accurate reporting - if (state.status() != CommandStatus.RUNNING) { - store.save(state.started()); - } else if (state.errorsCount() > 0) { - // Recovered after one or more transient errors — reset the streak. Single write, - // and only when there is something to reset, so healthy re-polls stay write-free. - store.save(state.clearErrors()); - } - return false; // Keep in queue - will retry and call checkStatus() + // Shutting down: this delivery was claimed before the shutdown began — usually + // microseconds before, though a task that queued behind other executor work reaches + // this same check later — and starting the handler now can only add work the drain has + // to wait out. One cloud call can be given a budget longer than the whole drain budget, + // so it cannot be waited out at all. Nothing has been mutated yet, so there is nothing + // to protect by proceeding and nothing to roll back: do not route. The delivery is + // settled as a retry, so it is redelivered on the claim cadence — to a replica that is + // not shutting down, or to this process after a restart — and runs from exactly this + // state. For a PROCESSING command that means the next poll lands on the claim cadence + // rather than checkStatusInterval, which only ever polls sooner, never later. + // Checked AFTER the terminal branch, so a stale message is still acked here rather than + // left to be redelivered into the next process. + if (draining) { + log.debug("Command not started - service is draining: id={}, status={}", state.id(), state.status()); + lease.retry(); + return; + } + // Route explicitly on the snapshot status, every value named. Terminal + // statuses are acked above, so reaching their arm is a routing bug, and an + // unmapped new status must never silently execute — both throw, landing in + // the retry-on-throw catch below. + final CommandResult result = switch (state.status()) { + // PROCESSING is the handler's own earlier declaration (an execute() that + // returned PROCESSING) — never the queue's impatience — so checkStatus() is + // only invoked on the async-work pattern it was written for. + case PROCESSING -> handler.checkStatus(command, state); + // PENDING executes; after a crash the state is still PENDING, and the + // message lease guarantees the crashed invocation is not still running on + // a live replica. + case PENDING -> handler.execute(command); + case SUCCEEDED, FAILED, CANCELLED -> throw new IllegalStateException("Terminal status must be acked before routing - id=" + state.id() + "; status=" + state.status()); + default -> throw new IllegalStateException("Unmapped command status: " + state.status() + " - id=" + state.id()); + }; + // A null result is a handler bug and must stay RETRYABLE: it lands in the + // retry-on-throw catch below, exactly as it did before the explicit routing + // (result.status() would have thrown). It must never double as a stale-message + // sentinel — acking would remove a live command's only message, stranding it + // with no retry driver left. + Objects.requireNonNull(result, () -> "Handler returned a null command result - id=" + state.id() + "; status=" + state.status()); + + if (result.status() == CommandStatus.PROCESSING) { + // The handler declared async work in flight: record the declaration, then + // schedule the next checkStatus() poll on the re-poll cadence — decoupled + // from the visibility timeout, which paces crash detection and error retries; + // tightening that clock must not multiply the polling load. + recordProcessingDeclaration(state); + lease.retryAfter(config.checkStatusInterval()); + return; } - // Terminal result (SUCCEEDED, FAILED, or CANCELLED) - // Apply the result to transition to terminal state - final CommandState newState = state.applyResult(result); - store.save(newState); - log.debug("Command completed: id={}, status={}", state.id(), newState.status()); - return true; // Remove from queue - processing complete - - } catch (Exception e) { - // A thrown handler is a transient/retryable condition, NOT a terminal command - // outcome: keep the message in the queue (return false) so the stream layer retains - // its lease and re-polls it. A genuine command failure is signalled by returning a - // FAILED CommandResult (handled above), never by throwing. Persisting FAILED + acking - // here would turn a transient/infra error (e.g. the Postgres pool closing during - // shutdown) into a permanent FAILED command while the domain entity is left - // non-terminal, stranding the work. Deciding a command has *permanently* failed is - // delegated to the domain layer that owns the entity state (see seqeralabs/sched#712). - log.error("Command processing errored, will retry: id={}", msg.commandId(), e); + // Terminal result, recorded via the CAS update: a refusal means the command went + // terminal underneath (a cancel won) — the redelivery acks on the terminal + // check, so RETRY loses nothing. + if (store.update(state.id(), s0 -> s0.applyResult(result))) { + log.debug("Command completed: id={}, status={}", state.id(), result.status()); + lease.ack(); + } + else { + settleUnrecordedResult(state, lease); + } + } + catch (Exception e) { + // Retry-on-throw (#890), now uniform: the state is unchanged, so the redelivery + // re-executes a PENDING command or re-polls a PROCESSING one. No rollback needed: + // there is no queue-invented PROCESSING to roll back. + log.error("Command processing errored, will retry: id={}", command.id(), e); recordError(state, e); - return false; // Keep in queue - redelivered / re-polled + lease.retry(); + } + finally { + // Backstop for non-Exception Throwables (OOME, NoClassDefFoundError out of + // handler code): a lease must never outlive its task. Idempotent — a no-op + // when a branch above already settled. + lease.retry(); } } /** - * Execute a command handler with a timeout. - * - *

Submits the handler's {@code execute()} method to a thread pool and waits - * up to {@code config.executeTimeout()} for completion. This prevents slow handlers from - * blocking the queue processor thread. + * Settle a delivery whose terminal-result write was refused. Two very different causes + * hide behind that refusal and must not read as one: a VERIFIED terminal (or missing) + * state means a cancel — or expiry — won underneath; the result is dropped by design and + * the stale message acks now instead of burning one more redelivery on the terminal + * pre-check. Anything else is the theoretical exhaustion of the CAS retry bound against + * a LIVE command — retried via redelivery, loudly: mislabeling it as terminal would hide + * a command looping on redelivery until its state TTL. + */ + private void settleUnrecordedResult(CommandState state, MessageLease lease) { + final CommandStatus status = store.findById(state.id()).map(CommandState::status).orElse(null); + if (status == null || status.isTerminal()) { + log.info("Command result not recorded, already terminal: id={}; status={}", state.id(), status); + lease.ack(); + return; + } + log.warn("Command result write did not converge, will retry: id={}; status={}", state.id(), status); + lease.retry(); + } + + /** + * Submit the handler task to the pool, registering it for exactly as long as it actually runs — + * the work a drain waits for, the admission cap gates on, and a shutdown report names. * - *

Timeout behavior: If the handler doesn't complete within the timeout, - * this method returns {@code null} but the handler continues executing in the - * background. The caller should mark the command as RUNNING and retry later - * via {@code checkStatus()}. + *

The two unregister sites are mutually exclusive, so one submission can never remove twice: + * the task's own {@code finally} runs iff the task ran, and the catch below runs iff the executor + * rejected the submission, in which case the task never ran at all. Without that catch a rejection + * leaks the entry for good: every later {@link #drain} reports false, the readiness indicator shows + * phantom activeTasks forever, and the admission cap starves the dispatcher. * - * @param handler The command handler to execute - * @param command The command with parameters - * @param

The command parameter type - * @param The command result type - * @return The result if completed within timeout, or {@code null} if timed out - * @throws RuntimeException if the handler throws an exception + * @param task the handler invocation to run on the executor + * @param detail how the invocation is named in {@link #activeCommandDetails()} */ - private CommandResult executeWithTimeout(CommandHandler handler, Command

command) { - // Submit handler execution to thread pool for async execution - final Future> future = executor.submit(() -> handler.execute(command)); - + private Future submitCounted(Runnable task, String detail) { + final long seq = inflightSeq.incrementAndGet(); + inflight.put(seq, detail); try { - // Block until result is available or timeout expires - return future.get(config.executeTimeout().toMillis(), TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - // Handler is taking longer than allowed - let it continue in background - // Caller will mark as RUNNING and poll via checkStatus() on retry - return null; - } catch (Exception e) { - // Handler threw an exception - cancel the future and propagate - future.cancel(true); - throw new RuntimeException("Command execution failed", e); + return executor.submit(() -> { + try { + task.run(); + } + finally { + inflight.remove(seq); + } + }); + } + catch (RuntimeException e) { + inflight.remove(seq); + throw e; + } + } + + /** + * How an in-flight invocation is named in {@link #activeCommandDetails()}: the command id + * carries the correlation a responder greps for, the type says what kind of work it is (the + * difference between an expected slow launch and something unexpected). + */ + private static String describe(CommandState state) { + return state.id() + "(" + state.type() + ")"; + } + + /** + * Record a handler's PROCESSING declaration against the persisted state. Note the two + * distinct subjects: the handler's result said PROCESSING (checked by the caller); + * what happens next depends on what the persisted state already says. The first + * declaration (state still PENDING — the execute() that just kicked off async work) + * writes the transition, which is what routes the next delivery to {@code checkStatus()}. + * Subsequent polls (state already PROCESSING) are write-free, except to clear a recovered + * error streak — and only when there is one, so healthy re-polls stay write-free. + */ + private void recordProcessingDeclaration(CommandState state) { + if (state.status() != CommandStatus.PROCESSING) { + markProcessing(state); + } + else if (state.errorsCount() > 0) { + store.update(state.id(), CommandState::clearErrors); + } + } + + /** + * Mark the command PROCESSING so the next delivery polls {@code checkStatus()} instead of + * running {@code execute()} again — used when a handler explicitly returned a PROCESSING + * result. A silently skipped write here is not benign: the message is redelivered with the state + * still PENDING, and a second execution starts while the async work it declared is in flight. With + * the compare-and-swap {@code update()} a refusal from mere contention no longer exists; the + * single retry covers only the theoretical exhaustion of the CAS bound, and a persistent miss + * is logged loudly. That warning is alert-worthy: any occurrence marks the + * duplicate-execution precondition actually firing. The refusal can also be the terminal + * guard (the command was cancelled mid-execution); that one is harmless — the redelivery is + * acked on the terminal check — so it is not warned about. + */ + private void markProcessing(CommandState state) { + // one retry only — it covers just the theoretical exhaustion of the CAS bound + for (int attempt = 0; attempt < 2; attempt++) { + if (store.update(state.id(), CommandState::toProcessing)) { + return; + } + } + final CommandStatus status = store.findById(state.id()).map(CommandState::status).orElse(null); + if (status == null || !status.isTerminal()) { + log.warn("Command PROCESSING mark not recorded - id={}, status={}; a redelivery may start a second execution", state.id(), status); } } @@ -356,19 +629,18 @@ private CommandResult executeWithTimeout(CommandHandler handler, */ private void recordError(CommandState state, Exception e) { try { - store.save(state.withError(rootMessage(e))); + store.update(state.id(), s0 -> s0.withError(rootMessage(e))); } catch (Exception fail) { log.warn("Failed to record command error state: id={}", state.id(), fail); } } /** - * The most specific message available for a processing error. {@link #executeWithTimeout} - * wraps a handler exception in a generic {@code RuntimeException("Command execution failed")}, - * so the root cause's message is recorded instead — otherwise every transient error on the - * execute() path would read "Command execution failed" and the field would be useless for - * diagnosing a retry storm. The checkStatus() path throws directly, where the root cause is - * the exception itself. + * The most specific message available for a processing error. Handlers may wrap the + * underlying failure in generic layers (e.g. {@code RuntimeException("Command execution + * failed")}), so the root cause's message is recorded instead — otherwise every transient + * error would read as the wrapper and the field would be useless for diagnosing a retry + * storm. An unwrapped exception is its own root cause. */ private static String rootMessage(Throwable e) { Throwable root = e; diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandState.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandState.java index 8a79a248..e4884431 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandState.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandState.java @@ -18,8 +18,10 @@ import java.time.Instant; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.micronaut.core.annotation.Nullable; +import io.seqera.data.store.state.VersionAware; /** * Persistent state of a command, stored as JSON in the database. @@ -34,6 +36,13 @@ * {@code error != null}. {@code modifiedAt} is refreshed on every state write, giving a * last-touched timestamp. * + *

{@code version} is the optimistic-concurrency write witness (see {@link VersionAware}): + * stamped by the state store on every write, injected on read, and carried unchanged through + * every transition so a compare-and-swap lands only when the entry was not written since the + * read the transition derives from. It is {@code @JsonIgnore}d — the store persists it in a + * frame outside the payload, which is the single source of truth — and never assigned by + * callers. + * * @param id command id * @param type command type discriminator * @param status current lifecycle status @@ -44,9 +53,11 @@ * @param errorsCount number of consecutive processing errors since the last successful * processing; reset to 0 on any successful transition or recovery * @param createdAt when the command was first submitted - * @param startedAt when the command first transitioned to RUNNING (nullable) + * @param startedAt when the command first transitioned to PROCESSING (nullable) * @param modifiedAt when the command state was last written (nullable for pre-existing records) * @param completedAt when the command reached a terminal state (nullable) + * @param version optimistic-concurrency version of the read this state derives from; 0 for a + * state never written through the store */ public record CommandState( String id, @@ -61,29 +72,30 @@ public record CommandState( Instant createdAt, @Nullable Instant startedAt, @Nullable Instant modifiedAt, - @Nullable Instant completedAt -) { + @Nullable Instant completedAt, + @JsonIgnore long version +) implements VersionAware { /** - * Create a new submitted command state. + * Create a new command state, pending its first processing. */ - public static CommandState submitted(String id, String type, Object params) { + public static CommandState create(String id, String type, Object params) { final Instant now = Instant.now(); return new CommandState( - id, type, CommandStatus.SUBMITTED, params, - null, null, 0, now, null, now, null + id, type, CommandStatus.PENDING, params, + null, null, 0, now, null, now, null, 0 ); } /** - * Transition to RUNNING status. A successful (non-throwing) transition, so the + * Transition to PROCESSING status. A successful (non-throwing) transition, so the * consecutive-error streak is reset. */ - public CommandState started() { + public CommandState toProcessing() { final Instant now = Instant.now(); return new CommandState( - id, type, CommandStatus.RUNNING, params, - result, error, 0, createdAt, now, now, completedAt + id, type, CommandStatus.PROCESSING, params, + result, error, 0, createdAt, now, now, completedAt, version ); } @@ -94,7 +106,7 @@ public CommandState completed(Object result) { final Instant now = Instant.now(); return new CommandState( id, type, CommandStatus.SUCCEEDED, params, - result, null, 0, createdAt, startedAt, now, now + result, null, 0, createdAt, startedAt, now, now, version ); } @@ -105,7 +117,7 @@ public CommandState failed(String error) { final Instant now = Instant.now(); return new CommandState( id, type, CommandStatus.FAILED, params, - null, error, errorsCount, createdAt, startedAt, now, now + null, error, errorsCount, createdAt, startedAt, now, now, version ); } @@ -116,7 +128,7 @@ public CommandState cancelled() { final Instant now = Instant.now(); return new CommandState( id, type, CommandStatus.CANCELLED, params, - null, null, 0, createdAt, startedAt, now, now + null, null, 0, createdAt, startedAt, now, now, version ); } @@ -128,7 +140,7 @@ public CommandState cancelled() { public CommandState withError(String message) { return new CommandState( id, type, status, params, - result, message, errorsCount + 1, createdAt, startedAt, Instant.now(), completedAt + result, message, errorsCount + 1, createdAt, startedAt, Instant.now(), completedAt, version ); } @@ -139,7 +151,20 @@ public CommandState withError(String message) { public CommandState clearErrors() { return new CommandState( id, type, status, params, - result, error, 0, createdAt, startedAt, Instant.now(), completedAt + result, error, 0, createdAt, startedAt, Instant.now(), completedAt, version + ); + } + + /** + * Create a copy of this state carrying the specified optimistic-concurrency version. + * Called by the state store, which injects on read the version of the stored entry — + * never by application code. + */ + @Override + public CommandState withVersion(long version) { + return new CommandState( + id, type, status, params, + result, error, errorsCount, createdAt, startedAt, modifiedAt, completedAt, version ); } diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandStatus.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandStatus.java index 8114e6fd..47423466 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandStatus.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/CommandStatus.java @@ -17,16 +17,26 @@ package io.seqera.data.command; +import com.fasterxml.jackson.annotation.JsonAlias; + /** * Status of a command in the queue. + * + *

{@code PENDING} and {@code PROCESSING} were named {@code SUBMITTED} and {@code RUNNING} + * before the queue vocabulary was aligned with {@code WorkQueue}. The {@link JsonAlias} + * annotations are what keep state written by the previous naming readable: {@code CommandState} + * is persisted as Jackson-encoded JSON with the status as a bare enum name, so an entry stored + * as {@code "SUBMITTED"} or {@code "RUNNING"} — by an older replica during a rolling deploy, or + * before it, for as long as the stored state lives — only deserializes because of them. They must + * never be removed. */ public enum CommandStatus { - /** Created, not yet submitted to queue */ + /** In queue, awaiting first processing */ + @JsonAlias("SUBMITTED") PENDING, - /** In queue, waiting for pickup */ - SUBMITTED, - /** Being executed */ - RUNNING, + /** Being processed by a handler */ + @JsonAlias("RUNNING") + PROCESSING, /** Completed successfully */ SUCCEEDED, /** Completed with error */ diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStore.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStore.java index 0209c4ac..569f8c1d 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStore.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStore.java @@ -17,6 +17,7 @@ package io.seqera.data.command.store; import java.util.Optional; +import java.util.function.UnaryOperator; import io.seqera.data.command.CommandState; @@ -28,12 +29,54 @@ public interface CommandStateStore { /** - * Save a command state. + * Store a command state, overwriting unconditionally. * - * @param state the command state to save + *

Create only. Because the write is unconditional, using this to transition an + * existing command discards whatever another writer stored in the meantime — a concurrent + * {@code cancel()} against a completing command loses one of the two outcomes. Use + * {@link #update(String, UnaryOperator)} for every transition; this method is for the initial + * {@code PENDING} record, where there is nothing to overwrite. + * + * @param state the command state to store */ void save(CommandState state); + /** + * Transition an existing command through a compare-and-swap read-modify-write, so a + * concurrent writer cannot discard the result. + * + *

The mutator is applied to the state as freshly read, never to a snapshot the caller + * read earlier; the write lands only if no other writer got in between, and a miss re-reads + * and re-applies (bounded by {@code CommandConfig#stateUpdateAttempts()}) — mere contention + * is absorbed internally and never surfaced. + * + *

Returns {@code false} rather than throwing when the transition did not happen: the + * command no longer exists, has already reached a terminal state — or, theoretically, the + * CAS retry bound was exhausted against a live command. Callers whose write is best-effort + * (error bookkeeping) may ignore the result; callers whose write is load-bearing must + * distinguish the terminal case from the exhausted one (re-read the status) and leave the + * work queued so it is retried. + * + *

The CAS witness is the version stamped by the store on every write and carried by the + * state through its transitions (see {@code VersionAware}) — never the serialized form of + * the state, so it holds across replicas whose JVMs serialize the same value differently, + * and adding a field to {@link CommandState} does not affect records written before the + * change (they are adopted by their first successful update). + * + *

Deployment note: a replica still running the pre-versioning code cannot update an + * entry a versioned replica has written — its byte comparison sees the version frame the store + * prepends to the stored form and never matches, while the read itself keeps succeeding + * silently (unknown properties are ignored). Old replicas therefore keep refusing every such + * transition — a missed PROCESSING mark makes redelivery re-run {@code execute()} instead of + * {@code checkStatus()}, duplicating the work — until every replica runs the versioned code. + * Validate multi-replica behavior only once the rollout is complete. + * + * @param commandId the command to transition + * @param mutator derives the new state from the current one + * @return {@code true} if the new state was stored + */ + boolean update(String commandId, UnaryOperator mutator); + /** * Find a command state by ID. * diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStoreFactory.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStoreFactory.java index 400f2e54..f7632042 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStoreFactory.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStoreFactory.java @@ -41,6 +41,6 @@ public class CommandStateStoreFactory { @Singleton public CommandStateStore commandStateStore(StateProvider provider) { final var encoder = new JacksonEncodingStrategy(){}; - return new CommandStateStoreImpl(provider, encoder, config.stateTtl()); + return new CommandStateStoreImpl(provider, encoder, config); } } diff --git a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStoreImpl.java b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStoreImpl.java index 43c55d79..33df9b37 100644 --- a/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStoreImpl.java +++ b/lib-cmd-queue-redis/src/main/java/io/seqera/data/command/store/CommandStateStoreImpl.java @@ -18,11 +18,15 @@ import java.time.Duration; import java.util.Optional; +import java.util.function.UnaryOperator; +import io.seqera.data.command.CommandConfig; import io.seqera.data.command.CommandState; -import io.seqera.data.store.state.AbstractStateStore; +import io.seqera.data.store.state.VersionedStateStore; import io.seqera.data.store.state.impl.StateProvider; import io.seqera.serde.encode.StringEncodingStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * State store implementation for command persistence using lib-data-store-state-redis. @@ -35,15 +39,25 @@ * * @author Paolo Di Tommaso */ -public class CommandStateStoreImpl extends AbstractStateStore implements CommandStateStore { +public class CommandStateStoreImpl extends VersionedStateStore implements CommandStateStore { + + private static final Logger log = LoggerFactory.getLogger(CommandStateStoreImpl.class); private static final String PREFIX = "cmd-state/v1"; private final Duration ttl; - public CommandStateStoreImpl(StateProvider provider, StringEncodingStrategy encodingStrategy, Duration ttl) { + /** Bound on the compare-and-swap retry loop in {@link #update} — see {@link CommandConfig#stateUpdateAttempts()}. */ + private final int updateAttempts; + + public CommandStateStoreImpl(StateProvider provider, StringEncodingStrategy encodingStrategy, CommandConfig config) { super(provider, encodingStrategy); - this.ttl = ttl; + this.ttl = config.stateTtl(); + this.updateAttempts = config.stateUpdateAttempts(); + // Fail fast: a non-positive bound would make every update() a silent no-op refusal + if (updateAttempts <= 0) { + throw new IllegalStateException("Command state update attempts must be positive - offending value: " + updateAttempts); + } } @Override @@ -66,4 +80,35 @@ public void save(CommandState state) { put(state.id(), state); } + /** + * Compare-and-swap read-modify-write. The mutator is applied to the freshly read state and + * the write lands only if the entry was not written since that read — the witness is the + * version the state carries from the read and preserves through its transitions (see + * {@code VersionedStateStore#replaceIf}, libseqera#107); on a miss the loop re-reads and + * re-applies, so mere contention is absorbed internally and never surfaced. A {@code false} + * return means the command is terminal or missing — plus the theoretical exhaustion of the + * retry bound, which callers already treat as "retry via redelivery". The entry TTL is + * refreshed on every write, matching {@code put()}. + */ + @Override + public boolean update(String commandId, UnaryOperator mutator) { + for (int i = 0; i < updateAttempts; i++) { + final CommandState current = get(commandId); + if (current == null || current.status().isTerminal()) { + return false; + } + final CommandState next = mutator.apply(current); + if (next == current) { + return true; // no-op mutator: stay write-free + } + // the witness is this loop's own read, whatever the mutator did to the version + if (replaceIf(commandId, next.withVersion(current.version()), ttl)) { + return true; + } + // CAS miss: another writer transitioned the state — re-read and re-apply + } + log.warn("Command state write did not converge - id={}", commandId); + return false; + } + } diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandQueueShowcaseTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandQueueShowcaseTest.groovy index 08fad2cf..2a6af3b0 100644 --- a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandQueueShowcaseTest.groovy +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandQueueShowcaseTest.groovy @@ -92,7 +92,7 @@ class CommandQueueShowcaseTest extends Specification { // ========================================================================= /** - * Demonstrates a long-running command that returns RUNNING status initially + * Demonstrates a long-running command that returns PROCESSING status initially * and completes asynchronously. The command queue periodically checks status * until the command reaches a terminal state. * @@ -128,8 +128,8 @@ class CommandQueueShowcaseTest extends Specification { sleep(500) def initialState = commandService.getState(commandId).orElseThrow() - then: 'command is in RUNNING state (async processing started)' - initialState.status() == CommandStatus.RUNNING + then: 'command is in PROCESSING state (async processing started)' + initialState.status() == CommandStatus.PROCESSING when: 'wait for async completion via periodic status checks' sleep(4000) @@ -476,7 +476,7 @@ class DataProcessingCommand implements Command { /** * Handler for data processing commands - executes asynchronously. - * Returns RUNNING immediately, then checkStatus() is called periodically + * Returns PROCESSING immediately, then checkStatus() is called periodically * until processing completes. */ class DataProcessingHandler implements CommandHandler { @@ -492,8 +492,8 @@ class DataProcessingHandler implements CommandHandler getProperties() { + return [ + 'command-queue.poll-interval' : '50ms', + 'workqueue.local.retry-delay' : '50ms', // plain retries stay fast... + 'command.check-status-interval' : '600ms', // ...but PROCESSING re-polls pace on the cadence + ] + } + + def setupSpec() { + commandService.registerHandler(new CommandHandler() { + @Override + String type() { 'poll-forever' } + @Override + CommandResult execute(Command command) { CommandResult.processing() } + @Override + CommandResult checkStatus(Command command, CommandState state) { + polls.incrementAndGet() + return CommandResult.processing() + } + }) + commandService.start() + } + + def cleanupSpec() { + commandService.stop() + } + + def 'a running command should be re-polled on the check-status interval, not the retry delay'() { + given: + def command = new TestCommand(TsidCreator.getTsid().toLowerCase(), 'poll-forever', new TestParams(1, 'x')) + + when: 'the command declares PROCESSING and is observed over a 2s window' + commandService.submit(command) + sleep 2_000 + + then: 'polls are paced by the 600ms cadence - not the 50ms retry delay (which would give ~30)' + polls.get() <= 5 + polls.get() >= 1 + } + +} diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceDrainBudgetTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceDrainBudgetTest.groovy new file mode 100644 index 00000000..7b106b8a --- /dev/null +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceDrainBudgetTest.groovy @@ -0,0 +1,150 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.command + +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +import com.github.f4b6a3.tsid.TsidCreator +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.micronaut.test.support.TestPropertyProvider +import jakarta.inject.Inject + +import spock.lang.Specification + +/** + * Covers how {@code drain()} divides its budget between its two waits. + * + *

Step 1 waits for the dispatcher to quiesce; step 2 waits for handler tasks already running + * on the executor. The deadline for step 2 is taken before step 1 runs, so whatever + * step 1 spends comes out of step 2's share. Handed the whole budget, a dispatcher that never + * quiesces leaves step 2 with nothing — its loop runs zero iterations and handler tasks + * mid-flight against the database get no grace at all, which is the one thing the drain exists + * to provide (#888). Both incomplete drains observed in production have exactly that shape, + * each reporting {@code dispatcherStopped=false} (#955). + * + *

{@link StallingCommandQueue} never quiesces, so step 1 always spends everything it is + * given — which is what makes step 2's remaining share observable. + * + * @author Paolo Di Tommaso + */ +@MicronautTest(packages = ["io.seqera.data.workqueue"], transactional = false, rebuildContext = true) +class CommandServiceDrainBudgetTest extends Specification implements TestPropertyProvider { + + @Inject + CommandService commandService + + @Override + Map getProperties() { + return [ + 'command-queue.poll-interval' : '100ms', + 'test.command-queue.stalling' : 'true' + ] + } + + def setup() { + StallingCommandQueue.FIRST_QUIESCE_BUDGET_MILLIS.set(-1) + StallingCommandQueue.STOPS_DURING_CLOSE.set(false) + } + + def 'drain should reserve a share of its budget for the in-flight wait'() { + given: 'a service with nothing in flight, so only the budget split is under test' + commandService.registerHandler(new SlowHandler(runFor: 0)) + commandService.start() + + when: + commandService.drain(Duration.ofSeconds(8)) + + then: 'step 1 was capped at a quarter, leaving the rest for the in-flight wait' + StallingCommandQueue.FIRST_QUIESCE_BUDGET_MILLIS.get() == 2_000 + } + + def 'a dispatcher that never quiesces should not consume the in-flight wait'() { + given: 'a handler that outlives step 1 but finishes well inside the whole budget' + def handler = new SlowHandler(runFor: 1_200) + commandService.registerHandler(handler) + commandService.start() + + and: + def command = new TestCommand(TsidCreator.getTsid().toLowerCase(), 'slow-drain', new TestParams(1, 'x')) + + when: 'the handler is running on the executor before the drain begins' + commandService.submit(command) + handler.entered.await(10, TimeUnit.SECONDS) + + and: + def begin = System.currentTimeMillis() + def drained = commandService.drain(Duration.ofSeconds(4)) + def elapsed = System.currentTimeMillis() - begin + + then: 'the drain waited for the handler rather than burning the whole budget in step 1' + handler.completed.get() + !handler.interrupted.get() + commandService.activeCommands() == 0 + // ~1.2s once step 2 gets its share; ~4s (the whole budget) when step 1 takes everything + elapsed < 2_500 + + and: 'it still reports incomplete, because the dispatcher never stopped' + !drained + } + + def 'a dispatcher that stops during close should be reported as a completed drain'() { + given: 'a dispatcher that outlives its quiesce budget but stops within the drain budget' + StallingCommandQueue.STOPS_DURING_CLOSE.set(true) + commandService.registerHandler(new SlowHandler(runFor: 0)) + commandService.start() + + when: + def drained = commandService.drain(Duration.ofSeconds(4)) + + then: 'slow is not stuck: everything settled inside the budget, so nothing was lost' + drained + } + + static class SlowHandler implements CommandHandler { + long runFor + final CountDownLatch entered = new CountDownLatch(1) + final AtomicBoolean completed = new AtomicBoolean(false) + final AtomicBoolean interrupted = new AtomicBoolean(false) + + @Override + String type() { 'slow-drain' } + + @Override + CommandResult execute(Command command) { + entered.countDown() + try { + Thread.sleep(runFor) + completed.set(true) + } + catch (InterruptedException e) { + interrupted.set(true) + Thread.currentThread().interrupt() + } + return CommandResult.success(new TestResult('done', command.params().value)) + } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + return CommandResult.processing() + } + } + +} diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceDrainTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceDrainTest.groovy new file mode 100644 index 00000000..e2e96470 --- /dev/null +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceDrainTest.groovy @@ -0,0 +1,309 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.command + +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +import com.github.f4b6a3.tsid.TsidCreator +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.micronaut.test.support.TestPropertyProvider +import io.seqera.data.command.store.CommandStateStore +import jakarta.inject.Inject + +import spock.lang.Specification +import spock.util.concurrent.PollingConditions +/** + * Covers {@code drain()} under task-settled delivery: every handler invocation — execute() + * calls AND checkStatus() polls — runs as a counted task on the blocking executor, and a + * shutdown must wait for all of them. That window is where a handler is still writing to a + * database whose connection pool is about to be closed. + * + *

checkStatus() polls used to run uncounted on the dispatcher thread; the message-lease + * model counts them in {@code inflight}, so drain no longer abandons an in-flight poll — + * the delta called out in docs/command-execution-guarantee-message-lease.md. + * + * @author Paolo Di Tommaso + */ +// rebuildContext: drain() releases the queue for good, so each feature needs its own +// CommandService rather than a context shared across the class +@MicronautTest(packages = ["io.seqera.data.workqueue"], transactional = false, rebuildContext = true) +class CommandServiceDrainTest extends Specification implements TestPropertyProvider { + + @Inject + CommandService commandService + + @Inject + CommandStateStore store + + @Override + Map getProperties() { + return [ + 'command-queue.poll-interval': '100ms' + ] + } + + def 'drain should wait for a handler task still running on the executor'() { + given: + def handler = new SlowHandler(runFor: 1_500) + commandService.registerHandler(handler) + commandService.start() + + and: + def command = new TestCommand(TsidCreator.getTsid().toLowerCase(), 'slow-drain', new TestParams(1, 'x')) + + when: 'the command is dispatched and its task keeps running past the DEFERRED hand-off' + commandService.submit(command) + handler.entered.await(10, TimeUnit.SECONDS) + + then: 'it is counted as in flight even though the dispatcher already moved on' + commandService.activeCommands() == 1 + + when: + def drained = commandService.drain(Duration.ofSeconds(10)) + + then: 'drain blocked until the handler finished, and it was never interrupted' + drained + handler.completed.get() + !handler.interrupted.get() + commandService.activeCommands() == 0 + } + + def 'drain should report incomplete and leave the count visible when the handler outlives the budget'() { + given: 'the handler is held open by the test, so the count cannot race the assertion' + def release = new CountDownLatch(1) + def handler = new SlowHandler(release: release) + commandService.registerHandler(handler) + commandService.start() + + and: + def command = new TestCommand(TsidCreator.getTsid().toLowerCase(), 'slow-drain', new TestParams(2, 'x')) + + when: + commandService.submit(command) + handler.entered.await(10, TimeUnit.SECONDS) + def drained = commandService.drain(Duration.ofMillis(300)) + + then: 'the caller is told the drain was incomplete rather than silently proceeding' + !drained + commandService.activeCommands() == 1 + + and: '''the survivor is identified, not merely counted: a drain that expires on a slow cloud + call is an expected shape, and only the id and type tell that apart from something + unexpected''' + commandService.activeCommandDetails() == ["${command.id()}(slow-drain)".toString()] + + cleanup: 'let the still-running handler finish so it does not outlive the test' + release.countDown() + handler.finished.await(10, TimeUnit.SECONDS) + } + + def 'should name an in-flight command by id and type, and forget it once the task ends'() { + given: 'the handler is held open by the test, so the register cannot race the assertion' + def release = new CountDownLatch(1) + def handler = new SlowHandler(release: release) + commandService.registerHandler(handler) + commandService.start() + + and: + def command = new TestCommand(TsidCreator.getTsid().toLowerCase(), 'slow-drain', new TestParams(6, 'x')) + + expect: 'an idle service names nothing' + commandService.activeCommandDetails() == [] + + when: + commandService.submit(command) + handler.entered.await(10, TimeUnit.SECONDS) + + then: + commandService.activeCommandDetails() == ["${command.id()}(slow-drain)".toString()] + + when: 'the handler is released and its task returns' + release.countDown() + handler.finished.await(10, TimeUnit.SECONDS) + + then: '''the entry is unregistered — a leak here would name a phantom command in every + later shutdown report, exactly when a responder can least afford a false lead''' + new PollingConditions(timeout: 10).eventually { + assert commandService.activeCommandDetails() == [] + assert commandService.activeCommands() == 0 + } + + cleanup: + commandService.drain(Duration.ofSeconds(10)) + } + + def 'should report several in-flight commands in a stable order'() { + given: 'two commands whose handler invocations are both held open by the test' + def release = new CountDownLatch(1) + def handler = new SlowHandler(release: release) + commandService.registerHandler(handler) + commandService.start() + + and: 'ids known in lexical order, so the expected report is unambiguous' + def ids = [TsidCreator.getTsid().toLowerCase(), TsidCreator.getTsid().toLowerCase()].sort() + + when: '''the lexically LARGER id is submitted first: the register hands out keys in + submission order, so an unsorted report would come back the wrong way round''' + commandService.submit(new TestCommand(ids[1], 'slow-drain', new TestParams(7, 'x'))) + commandService.submit(new TestCommand(ids[0], 'slow-drain', new TestParams(8, 'x'))) + + then: 'a human reading the shutdown log gets the same order every time' + new PollingConditions(timeout: 10).eventually { + assert commandService.activeCommandDetails() == ["${ids[0]}(slow-drain)".toString(), "${ids[1]}(slow-drain)".toString()] + } + + cleanup: + release.countDown() + handler.finished.await(10, TimeUnit.SECONDS) + commandService.drain(Duration.ofSeconds(10)) + } + + def 'drain should not exceed its budget when in-flight work never finishes'() { + given: 'a handler that outlives the whole drain budget' + def handler = new SlowHandler(runFor: 5_000) + commandService.registerHandler(handler) + commandService.start() + + and: + def command = new TestCommand(TsidCreator.getTsid().toLowerCase(), 'slow-drain', new TestParams(3, 'x')) + + when: + commandService.submit(command) + handler.entered.await(10, TimeUnit.SECONDS) + def begin = System.currentTimeMillis() + commandService.drain(Duration.ofMillis(500)) + def elapsed = System.currentTimeMillis() - begin + + then: '''close() must be bounded by what is left of drain's budget, not start its own timer. + Before this fix it added a further closeTimeout() (10s) plus a 1s join, which would + blow past a container graceful-shutdown grace period and get us hard-stopped + mid-drain — the very thing draining exists to avoid.''' + elapsed < 3_000 + + cleanup: + handler.finished.await(10, TimeUnit.SECONDS) + } + + def 'drain should count and wait for an in-flight checkStatus poll'() { + given: 'a PROCESSING command whose status poll blocks on the executor' + def release = new CountDownLatch(1) + def handler = new BlockingCheckStatusHandler(release: release) + commandService.registerHandler(handler) + commandService.start() + + and: + def command = new TestCommand( + TsidCreator.getTsid().toLowerCase(), + 'blocking-check-status', + new TestParams(4, 'x')) + + when: + commandService.submit(command) + handler.entered.await(10, TimeUnit.SECONDS) + + then: 'the poll is a counted task, no longer invisible dispatcher work' + commandService.activeCommands() == 1 + + when: 'the poll is released while the drain is waiting on it' + Thread.start { + sleep 500 + release.countDown() + } + def drained = commandService.drain(Duration.ofSeconds(10)) + + then: 'drain waited for the poll to settle instead of abandoning it' + drained + handler.finished.await(10, TimeUnit.SECONDS) + commandService.activeCommands() == 0 + } + + def 'drain should be a no-op when the service was never started'() { + expect: + commandService.drain(Duration.ofSeconds(1)) + commandService.activeCommands() == 0 + } + + static class SlowHandler implements CommandHandler { + /** Fixed run time, used when {@link #release} is null. */ + long runFor + /** When set, the handler blocks until the test releases it — no wall-clock guessing. */ + CountDownLatch release + final CountDownLatch entered = new CountDownLatch(1) + final CountDownLatch finished = new CountDownLatch(1) + final AtomicBoolean completed = new AtomicBoolean(false) + final AtomicBoolean interrupted = new AtomicBoolean(false) + + @Override + String type() { 'slow-drain' } + + @Override + CommandResult execute(Command command) { + entered.countDown() + try { + if (release != null) + release.await(30, TimeUnit.SECONDS) + else + Thread.sleep(runFor) + completed.set(true) + } + catch (InterruptedException e) { + interrupted.set(true) + Thread.currentThread().interrupt() + } + finally { + finished.countDown() + } + return CommandResult.success(new TestResult('done', command.params().value)) + } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + return CommandResult.processing() + } + } + + static class BlockingCheckStatusHandler implements CommandHandler { + CountDownLatch release + final CountDownLatch entered = new CountDownLatch(1) + final CountDownLatch finished = new CountDownLatch(1) + + @Override + CommandResult execute(Command command) { + return CommandResult.processing() + } + + @Override + String type() { 'blocking-check-status' } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + entered.countDown() + try { + release.await(30, TimeUnit.SECONDS) + } + finally { + finished.countDown() + } + return CommandResult.success(new TestResult('polled', command.params().value)) + } + } + +} diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceLeaseRedisTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceLeaseRedisTest.groovy new file mode 100644 index 00000000..c9066e9b --- /dev/null +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceLeaseRedisTest.groovy @@ -0,0 +1,451 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.command + +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger + +import com.github.f4b6a3.tsid.TsidCreator +import io.micronaut.context.ApplicationContext +import io.seqera.data.command.store.CommandStateStoreImpl +import io.seqera.data.workqueue.WorkQueue +import io.seqera.data.workqueue.redis.RedisWorkQueue +import io.seqera.data.workqueue.redis.RedisWorkQueueConfig +import io.seqera.data.store.state.impl.RedisStateProvider +import io.seqera.fixtures.redis.RedisTestContainer +import io.seqera.serde.jackson.JacksonEncodingStrategy +import redis.clients.jedis.JedisPool +import spock.lang.Specification +/** + * Container proof of the queue layer under the message-lease model + * (docs/command-execution-guarantee-message-lease.md), against a real Redis with a + * SHORT visibility timeout: a handler outliving the visibility timeout is never executed twice + * across replicas, a crashed replica's command is re-executed elsewhere, PROCESSING is + * only ever the handler's own declaration polled on the claim cadence, a throwing + * handler is retried with the error streak visible, and the admission cap bounds + * concurrent handler tasks. + * + * @author Paolo Di Tommaso + */ +class CommandServiceLeaseRedisTest extends Specification implements RedisTestContainer { + + static final Duration VISIBILITY_TIMEOUT = Duration.ofSeconds(1) + + static class ShortVisibilityQueueConfig implements RedisWorkQueueConfig { + @Override + String getDefaultConsumerGroupName() { 'cmd-lease-test-group' } + @Override + Duration getVisibilityTimeout() { VISIBILITY_TIMEOUT } + @Override + Duration getConsumerWarnTimeout() { Duration.ofSeconds(10) } + } + + static class LeaseCommandConfig implements CommandConfig { + int cap = 25 + @Override + Duration pollInterval() { Duration.ofMillis(100) } + @Override + int maxConcurrency() { cap } + @Override + Duration stateTtl() { Duration.ofHours(1) } + @Override + Duration checkStatusInterval() { VISIBILITY_TIMEOUT } // at the floor: re-polls on the claim cadence + } + + /** Queue with a per-test name, so each feature gets its own Redis stream. */ + static class LeaseTestQueue extends CommandQueue { + private final String qname + LeaseTestQueue(WorkQueue target, String qname) { + super(target) + this.qname = qname + } + @Override + protected String name() { qname } + @Override + protected Duration pollInterval() { Duration.ofMillis(100) } + } + + /** One scheduler replica: its own queue consumer, dispatcher and executor, shared Redis. */ + static class Replica { + RedisWorkQueue redisQueue + CommandQueue queue + CommandServiceImpl service + ExecutorService executor + + /** Simulate a replica death: the dispatcher stops claiming and heartbeats stop. */ + void kill() { + queue.awaitQuiescent(Duration.ofSeconds(5)) + redisQueue.@renewalScheduler.shutdownNow() + } + + void shutdown() { + service.stop() + redisQueue.@renewalScheduler.shutdownNow() + executor.shutdownNow() + } + } + + ApplicationContext context + CommandStateStoreImpl store + List replicas = [] + + def setup() { + context = ApplicationContext.run('test', 'redis') + sleep(500) // workaround to wait for Redis connection, as in RedisStateProviderTest upstream + store = new CommandStateStoreImpl( + context.getBean(RedisStateProvider), + new JacksonEncodingStrategy() {}, + new LeaseCommandConfig()) + } + + def cleanup() { + replicas.each { it.shutdown() } + replicas.clear() + context.stop() + } + + private Replica newReplica(String queueName, LeaseCommandConfig cfg = new LeaseCommandConfig()) { + final redisQueue = new RedisWorkQueue() + redisQueue.@pool = context.getBean(JedisPool) + redisQueue.@config = new ShortVisibilityQueueConfig() + redisQueue.create() + final queue = new LeaseTestQueue(redisQueue, queueName) + final executor = Executors.newCachedThreadPool() + final service = new CommandServiceImpl(config: cfg, store: store, queue: queue, executor: executor) + final replica = new Replica(redisQueue: redisQueue, queue: queue, service: service, executor: executor) + replicas << replica + return replica + } + + private static String uniqueName() { + return "cmd-lease-${TsidCreator.getTsid().toLowerCase()}" + } + + private static TestCommand command(String type) { + return new TestCommand(TsidCreator.getTsid().toLowerCase(), type, new TestParams(1, 'x')) + } + + private CommandState awaitStatus(String commandId, CommandStatus expected, long timeoutMillis = 15_000) { + final deadline = System.currentTimeMillis() + timeoutMillis + CommandState state = null + while (System.currentTimeMillis() < deadline) { + state = store.findById(commandId).orElse(null) + if (state?.status() == expected) + return state + sleep 100 + } + return state + } + + def 'a handler running well past the visibility timeout should execute exactly once across two replicas'() { + given: 'two replicas sharing the queue, and a handler sleeping 3x the visibility timeout' + def queueName = uniqueName() + def executions = new AtomicInteger() + def handler = new SlowExecuteHandler(executions: executions, runFor: VISIBILITY_TIMEOUT.toMillis() * 3) + def replica1 = newReplica(queueName) + def replica2 = newReplica(queueName) + [replica1, replica2].each { + it.service.registerHandler(handler) + it.service.start() + } + + when: 'a command outliving the visibility timeout is submitted' + def cmd = command('slow-execute') + replica1.service.submit(cmd) + def state = awaitStatus(cmd.id(), CommandStatus.SUCCEEDED) + + then: 'the command completed with the handler result' + state.status() == CommandStatus.SUCCEEDED + + when: 'a grace period (1.5x the visibility timeout) passes in which a stalled entry would surface as a second execution' + sleep 1_500 + + then: 'the message lease kept the entry invisible - exactly one execution' + executions.get() == 1 + } + + def 'a command whose execution died with its replica should be re-executed exactly once elsewhere'() { + given: 'replica1 whose handler blocks forever, replica2 whose handler succeeds' + def queueName = uniqueName() + def blockedRelease = new CountDownLatch(1) + def blockedHandler = new BlockingExecuteHandler(release: blockedRelease) + def recoveryHandler = new CountingHandler() + def replica1 = newReplica(queueName) + def replica2 = newReplica(queueName) + replica1.service.registerHandler(blockedHandler) + replica2.service.registerHandler(recoveryHandler) + replica1.service.start() + + when: 'replica1 claims the command and then dies mid-execute' + def cmd = command('crash-test') + replica1.service.submit(cmd) + blockedHandler.entered.await(10, TimeUnit.SECONDS) + replica1.kill() + + and: 'replica2 comes up and polls the shared queue' + replica2.service.start() + def state = awaitStatus(cmd.id(), CommandStatus.SUCCEEDED) + + then: 'the entry idled out, redelivered, and completed on replica2' + state.status() == CommandStatus.SUCCEEDED + + and: 'the state was still PENDING, so recovery was execute() again - exactly once' + recoveryHandler.executions.get() == 1 + recoveryHandler.statusChecks.get() == 0 + + cleanup: + blockedRelease.countDown() + } + + def 'a handler-declared PROCESSING should be polled via checkStatus on the claim cadence'() { + given: + def queueName = uniqueName() + def handler = new AsyncWorkHandler() + def replica = newReplica(queueName) + replica.service.registerHandler(handler) + replica.service.start() + + when: 'the command is submitted and execute() is still in flight' + def cmd = command('async-work') + replica.service.submit(cmd) + handler.entered.await(10, TimeUnit.SECONDS) + + and: 'the state is sampled for as long as execute() has not returned' + def samples = [] + while (handler.executeReturnedAt == 0) { + def status = store.findById(cmd.id()).orElseThrow().status() + if (handler.executeReturnedAt == 0) + samples << status + sleep 100 + } + + then: 'no PROCESSING was fabricated before the handler declared it' + !samples.isEmpty() + samples.every { it == CommandStatus.PENDING } + + when: + def state = awaitStatus(cmd.id(), CommandStatus.SUCCEEDED) + + then: 'the handler declaration was recorded and the poll completed the command' + state.status() == CommandStatus.SUCCEEDED + state.startedAt() != null + handler.statusChecks.get() == 1 + + and: 'the checkStatus poll rode the claim cadence, not a busy loop - the idle clock runs from the last heartbeat, up to one renewal period before execute() returned' + handler.firstCheckAt - handler.executeReturnedAt >= VISIBILITY_TIMEOUT.toMillis() / 2 + } + + def 'an execute that throws twice should be redelivered each claim cycle and finally succeed'() { + given: 'a handler that throws on the first two attempts' + def queueName = uniqueName() + def handler = new FlakyHandler(failures: 2) + def replica = newReplica(queueName) + replica.service.registerHandler(handler) + replica.service.start() + + when: 'the command is submitted and the store is sampled while it retries' + def cmd = command('flaky-redis') + replica.service.submit(cmd) + def maxErrors = 0 + CommandState state = null + def deadline = System.currentTimeMillis() + 20_000 + while (System.currentTimeMillis() < deadline) { + state = store.findById(cmd.id()).orElse(null) + maxErrors = Math.max(maxErrors, state ? state.errorsCount() : 0) + if (state?.status() == CommandStatus.SUCCEEDED) + break + sleep 100 + } + + then: 'each throw was retried on the claim cadence until the third attempt succeeded' + state.status() == CommandStatus.SUCCEEDED + handler.attempts.get() == 3 + + and: 'the error streak was visible while the command retried' + maxErrors >= 1 + + and: 'no PROCESSING was ever fabricated - the state stayed PENDING across throws' + state.startedAt() == null + } + + def 'the admission cap should bound concurrent handler tasks while all commands complete'() { + given: 'a replica capped at 2 in-flight tasks and 5 queued slow commands' + def queueName = uniqueName() + def handler = new ConcurrencyTrackingHandler(runFor: 500) + def replica = newReplica(queueName, new LeaseCommandConfig(cap: 2)) + replica.service.registerHandler(handler) + replica.service.start() + + when: + def commands = (1..5).collect { command('concurrency-probe') } + commands.each { replica.service.submit(it) } + def states = commands.collect { awaitStatus(it.id(), CommandStatus.SUCCEEDED, 20_000) } + + then: 'all 5 completed' + states.every { it.status() == CommandStatus.SUCCEEDED } + + and: 'never more than 2 handler invocations ran concurrently' + handler.maxConcurrent.get() <= 2 + handler.executions.get() == 5 + } + + // ------------------------------------------------------------------ + // handlers + // ------------------------------------------------------------------ + + static class SlowExecuteHandler implements CommandHandler { + AtomicInteger executions + long runFor + + @Override + String type() { 'slow-execute' } + + @Override + CommandResult execute(Command command) { + executions.incrementAndGet() + sleep runFor + return CommandResult.success(new TestResult('slow done', command.params().value)) + } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + return CommandResult.processing() + } + } + + static class BlockingExecuteHandler implements CommandHandler { + CountDownLatch release + final CountDownLatch entered = new CountDownLatch(1) + + @Override + String type() { 'crash-test' } + + @Override + CommandResult execute(Command command) { + entered.countDown() + release.await(60, TimeUnit.SECONDS) + return CommandResult.success(new TestResult('too late', command.params().value)) + } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + return CommandResult.processing() + } + } + + static class CountingHandler implements CommandHandler { + final AtomicInteger executions = new AtomicInteger() + final AtomicInteger statusChecks = new AtomicInteger() + + @Override + String type() { 'crash-test' } + + @Override + CommandResult execute(Command command) { + executions.incrementAndGet() + return CommandResult.success(new TestResult('recovered', command.params().value)) + } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + statusChecks.incrementAndGet() + return CommandResult.processing() + } + } + + static class AsyncWorkHandler implements CommandHandler { + final CountDownLatch entered = new CountDownLatch(1) + final AtomicInteger statusChecks = new AtomicInteger() + volatile long executeReturnedAt = 0 + volatile long firstCheckAt = 0 + + @Override + String type() { 'async-work' } + + @Override + CommandResult execute(Command command) { + entered.countDown() + // longer than the old 1s execute-timeout: the queue must NOT lose patience + sleep 1_200 + executeReturnedAt = System.currentTimeMillis() + return CommandResult.processing() + } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + if (statusChecks.incrementAndGet() == 1) + firstCheckAt = System.currentTimeMillis() + return CommandResult.success(new TestResult('async done', command.params().value)) + } + } + + static class FlakyHandler implements CommandHandler { + int failures + final AtomicInteger attempts = new AtomicInteger() + + @Override + String type() { 'flaky-redis' } + + @Override + CommandResult execute(Command command) { + final n = attempts.incrementAndGet() + if (n <= failures) + throw new RuntimeException("boom-$n") + return CommandResult.success(new TestResult('finally', command.params().value)) + } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + return CommandResult.processing() + } + } + + static class ConcurrencyTrackingHandler implements CommandHandler { + long runFor + final AtomicInteger current = new AtomicInteger() + final AtomicInteger maxConcurrent = new AtomicInteger() + final AtomicInteger executions = new AtomicInteger() + + @Override + String type() { 'concurrency-probe' } + + @Override + CommandResult execute(Command command) { + final now = current.incrementAndGet() + maxConcurrent.updateAndGet { Math.max(it, now) } + try { + sleep runFor + } + finally { + current.decrementAndGet() + } + executions.incrementAndGet() + return CommandResult.success(new TestResult('done', command.params().value)) + } + + @Override + CommandResult checkStatus(Command command, CommandState state) { + return CommandResult.processing() + } + } + +} diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceSafetyTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceSafetyTest.groovy new file mode 100644 index 00000000..d7b26b96 --- /dev/null +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceSafetyTest.groovy @@ -0,0 +1,298 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.command + +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutionException +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.RejectedExecutionException +import java.util.concurrent.TimeUnit + +import io.seqera.data.command.store.CommandStateStore +import io.seqera.data.workqueue.MessageConsumer +import io.seqera.data.workqueue.MessageLease +import spock.lang.Specification +/** + * Guards the silent-failure seams in command processing, plus the cooperative refusal that keeps a + * shutdown bounded. + * + *

The in-flight counter must never leak: a permanently over-counted value makes every later + * {@code drain()} report false, the readiness indicator show phantom activeTasks forever, and the + * admission cap starve the dispatcher. + * + *

The PROCESSING mark declared by a handler must not be silently skipped: a redelivery with the + * state still PENDING starts a second execution while the async work it declared is in flight. + * + *

And a delivery claimed just before a shutdown must not be routed at all — starting work the + * shutdown budget cannot wait out is the failure lever 1 of #955 closes — while a stale delivery is + * still acked and a restarted service resumes routing. + * + * @author Paolo Di Tommaso + */ +class CommandServiceSafetyTest extends Specification { + + def 'a rejected handler submission should settle as RETRY without leaking the in-flight count'() { + given: 'an executor already shut down, as during a non-graceful teardown' + def store = Mock(CommandStateStore) + def executor = Mock(ExecutorService) + def lease = Mock(MessageLease) + def service = new CommandServiceImpl(store: store, executor: executor) + service.registerHandler(new TestCommandHandler()) + def state = CommandState.create('cmd-reject', 'test', new TestParams(1, 'fast')) + + when: + def decision = service.processCommand(CommandMsg.of('cmd-reject', 'test'), lease) + + then: + 1 * store.findById('cmd-reject') >> Optional.of(state) + 1 * executor.submit(_) >> { throw new RejectedExecutionException('shutting down') } + + and: 'nothing ran and nothing owns the lease: the claim cycle re-delivers' + decision == MessageConsumer.Decision.RETRY + 0 * lease._ + service.activeCommands() == 0 + + and: '''nor does the rejection leak the entry: one stranded here would name a command that + never ran in every later shutdown report AND hold the admission cap down forever''' + service.activeCommandDetails() == [] + } + + def 'a throwing task should decrement the in-flight count exactly once'() { + given: 'a real executor, so the task genuinely runs and its finally-decrement fires' + def executor = Executors.newSingleThreadExecutor() + def service = new CommandServiceImpl(executor: executor) + + when: 'the failure surfaces via the future, never out of submit(), so only one site decrements' + def future = service.submitCounted({ throw new IllegalStateException('boom') } as Runnable, 'cmd-boom(test)') + future.get(5, TimeUnit.SECONDS) + + then: + thrown(ExecutionException) + and: '''the entry is removed on the throwing path too, not only on a clean return — and since + the count IS the register, a leak here would inflate both at once''' + service.activeCommands() == 0 + service.activeCommandDetails() == [] + + cleanup: + executor.shutdownNow() + } + + def 'a drain should report what was in flight at its deadline, not after the stream closed'() { + given: '''a task that is still running when the budget expires and finishes while the stream is + being released. close() gets whatever is left of the budget, so that window is real — + and reading the register after it would tell the caller the drain succeeded''' + def release = new CountDownLatch(1) + def finished = new CountDownLatch(1) + def executor = Executors.newSingleThreadExecutor() + def queue = Mock(CommandQueue) + def service = new CommandServiceImpl(queue: queue, executor: executor, config: Stub(CommandConfig)) + + and: 'the handler task blocks until close() lets it go' + service.start() + service.submitCounted({ release.await(10, TimeUnit.SECONDS); finished.countDown() } as Runnable, 'cmd-slow(test)') + + when: + def drained = service.drain(Duration.ofMillis(200)) + + then: 'the dispatcher stopped cleanly, so only the in-flight task can make this incomplete' + 1 * queue.awaitQuiescent(_) >> true + + and: 'the task finishes DURING close, exactly the window a post-close read would miss' + 1 * queue.close(_) >> { + release.countDown() + finished.await(10, TimeUnit.SECONDS) + } + + and: '''the caller is told the truth about its deadline: work outlived the budget, even though + nothing is in flight by the time drain() returns''' + !drained + + cleanup: + executor.shutdownNow() + } + + def 'the PROCESSING mark should be retried once when the first write is contended'() { + given: + def store = Mock(CommandStateStore) + def service = new CommandServiceImpl(store: store) + def state = CommandState.create('cmd-mark', 'test', new TestParams(2, 'fast')) + + when: + service.markProcessing(state) + + then: 'the first write is contended, the retry lands, and no status lookup is needed' + 2 * store.update('cmd-mark', _) >>> [false, true] + 0 * store.findById(_) + } + + def 'a persistently missed PROCESSING mark should be checked against the terminal guard'() { + given: + def store = Mock(CommandStateStore) + def service = new CommandServiceImpl(store: store) + def state = CommandState.create('cmd-miss', 'test', new TestParams(3, 'fast')) + + when: + service.markProcessing(state) + + then: 'both writes fail, so the status is looked up to tell contention from cancellation' + 2 * store.update('cmd-miss', _) >> false + 1 * store.findById('cmd-miss') >> Optional.of(state) + } + + def 'a refused result write should ack once the command is verified terminal'() { + given: 'a handler task whose terminal result write is refused because a cancel won underneath' + def store = Mock(CommandStateStore) + def lease = Mock(MessageLease) + def service = new CommandServiceImpl(store: store) + def params = new TestParams(1, 'fast') + def state = CommandState.create('cmd-refused-terminal', 'test', params) + def command = new TestCommand('cmd-refused-terminal', 'test', params) + + when: + service.runCommand(command, state, new TestCommandHandler(), lease) + + then: 'the refusal is verified against the store, not assumed' + 1 * store.update('cmd-refused-terminal', _) >> false + 1 * store.findById('cmd-refused-terminal') >> Optional.of(state.cancelled()) + and: 'the stale message settles now, not one redelivery later' + 1 * lease.ack() + } + + def 'a delivery claimed just before a drain should not be routed to the handler'() { + given: '''the window lever 1 of #955 closes: the dispatcher claimed this message just before + the drain began, and the handler is about to start a cloud call whose own timeout + is longer than the whole drain budget''' + def store = Mock(CommandStateStore) + def lease = Mock(MessageLease) + def handler = Mock(CommandHandler) + def params = new TestParams(1, 'fast') + def state = CommandState.create('cmd-draining', 'test', params) + def command = new TestCommand('cmd-draining', 'test', params) + def service = new CommandServiceImpl(store: store) + + when: '''the shutdown is signalled through the real entry point, not by setting the flag: + that is what proves drain() arms the refusal. Never started, so drain() takes its + early return and waits for nothing''' + service.drain(Duration.ofSeconds(1)) + service.runCommand(command, state, handler, lease) + + then: 'the handler is never entered, so the drain has nothing new to wait out' + 0 * handler.execute(_) + + and: 'no state is written: there is nothing to roll back, and nothing to explain later' + 0 * store._ + + and: '''settled for redelivery, never acked — a live command whose only message was acked + would be stranded with nothing left to advance it''' + (1.._) * lease.retry() + 0 * lease.ack() + } + + def 'a PROCESSING delivery should not be polled once stop() has signalled the shutdown'() { + given: 'a command whose handler declared async work, so this delivery routes to checkStatus' + def store = Mock(CommandStateStore) + def lease = Mock(MessageLease) + def handler = Mock(CommandHandler) + def params = new TestParams(2, 'fast') + def state = CommandState.create('cmd-draining-poll', 'test', params).toProcessing() + def command = new TestCommand('cmd-draining-poll', 'test', params) + def service = new CommandServiceImpl(store: store) + + when: 'stop() is the other arming point, and it signals even when it has nothing to stop' + service.stop() + service.runCommand(command, state, handler, lease) + + then: '''the poll is refused too, not only execute(): a poll can reach a cloud call of its + own, and the guard sits before the routing switch precisely so no status decides it''' + 0 * handler.checkStatus(_, _) + 0 * handler.execute(_) + and: + (1.._) * lease.retry() + 0 * lease.ack() + } + + def 'a terminal delivery should still be acked while shutting down'() { + given: 'a stale message for a command that already completed' + def store = Mock(CommandStateStore) + def lease = Mock(MessageLease) + def handler = Mock(CommandHandler) + def params = new TestParams(3, 'fast') + def state = CommandState.create('cmd-draining-stale', 'test', params).completed('done') + def command = new TestCommand('cmd-draining-stale', 'test', params) + def service = new CommandServiceImpl(store: store) + + when: + service.drain(Duration.ofSeconds(1)) + service.runCommand(command, state, handler, lease) + + then: '''the shutdown check sits AFTER the terminal branch on purpose: leaving a dead entry + to be redelivered into the next process buys nothing, and acking it costs nothing''' + 1 * lease.ack() + 0 * handler.execute(_) + 0 * handler.checkStatus(_, _) + } + + def 'a service started again should route deliveries once more'() { + given: 'a service that has been stopped, so the shutdown signal is set' + def store = Mock(CommandStateStore) + def queue = Mock(CommandQueue) + def lease = Mock(MessageLease) + def handler = Mock(CommandHandler) + def params = new TestParams(4, 'fast') + def state = CommandState.create('cmd-restarted', 'test', params) + def command = new TestCommand('cmd-restarted', 'test', params) + def service = new CommandServiceImpl(store: store, queue: queue) + + when: + service.start() + service.stop() + service.start() + service.runCommand(command, state, handler, lease) + + then: '''the signal is cleared on start, not latched for the life of the JVM — a service + that refused work forever after one stop() would process nothing at all''' + 1 * handler.execute(_) >> CommandResult.success(new TestResult('ok', 4)) + + and: 'the invocation completes cleanly through the terminal-result path' + 1 * store.update('cmd-restarted', _) >> true + 1 * lease.ack() + } + + def 'a refused result write against a live command should retry, never read as terminal'() { + given: 'the CAS bound exhausted while the command stays non-terminal - NOT a cancel' + def store = Mock(CommandStateStore) + def lease = Mock(MessageLease) + def service = new CommandServiceImpl(store: store) + def params = new TestParams(2, 'fast') + def state = CommandState.create('cmd-refused-live', 'test', params) + def command = new TestCommand('cmd-refused-live', 'test', params) + + when: + service.runCommand(command, state, new TestCommandHandler(), lease) + + then: 'the re-read shows a live command: retried via redelivery, never acked' + 1 * store.update('cmd-refused-live', _) >> false + 1 * store.findById('cmd-refused-live') >> Optional.of(state.toProcessing()) + and: + (1.._) * lease.retry() + 0 * lease.ack() + } + +} diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceTest.groovy index 8ccaad65..44b80e8f 100644 --- a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceTest.groovy +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/CommandServiceTest.groovy @@ -32,7 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger /** * End-to-end tests for the CommandService. */ -@MicronautTest(packages = ["io.seqera.data.stream"], transactional = false) +@MicronautTest(packages = ["io.seqera.data.workqueue"], transactional = false) @TestInstance(TestInstance.Lifecycle.PER_CLASS) class CommandServiceTest extends Specification implements TestPropertyProvider { @@ -111,8 +111,11 @@ class CommandServiceTest extends Specification implements TestPropertyProvider { } def 'should cancel pending command'() { - given: - def params = new TestParams(42, 'fast') + given: "a 'slow' command, so the cancel cannot lose the race against the dispatcher" + // 'slow' execute() declares PROCESSING - a non-terminal write - so cancel() succeeds + // regardless of interleaving; a 'fast' command may reach SUCCEEDED before cancel() + // runs, which refuses the cancel correctly (flaky on CI) + def params = new TestParams(42, 'slow') def command = new TestCommand(TsidCreator.getTsid().toLowerCase(), 'test', params) when: 'command is submitted and immediately cancelled' @@ -159,8 +162,8 @@ class CommandServiceTest extends Specification implements TestPropertyProvider { sleep(500) def state = commandService.getState(command.id()).orElseThrow() - then: 'status is RUNNING' - state.status() == CommandStatus.RUNNING + then: 'status is PROCESSING' + state.status() == CommandStatus.PROCESSING when: 'wait for periodic checker' sleep(3000) @@ -210,6 +213,28 @@ class CommandServiceTest extends Specification implements TestPropertyProvider { state.modifiedAt() != null } + def 'a null handler result should stay retryable, never ack the message'() { + given: 'a handler that returns null - a handler bug, not a terminal outcome' + commandService.registerHandler(new CommandHandler() { + @Override + String type() { 'null-result' } + @Override + CommandResult execute(Command command) { null } + @Override + CommandResult checkStatus(Command command, CommandState state) { null } + }) + def command = new TestCommand(TsidCreator.getTsid().toLowerCase(), 'null-result', new TestParams(1, 'x')) + + when: + commandService.submit(command) + sleep(2000) + def state = commandService.getState(command.id()).orElseThrow() + + then: 'the command is still live and visibly retrying - the message was never acked' + !state.status().isTerminal() + state.errorsCount() >= 1 + } + def 'should handle unknown command type'() { given: def params = new TestParams(42, 'fast') @@ -304,7 +329,7 @@ class TestCommandHandler implements CommandHandler { if (params.mode == 'slow') { startTime = Instant.now() - return CommandResult.running() + return CommandResult.processing() } def result = new TestResult('Processed', params.value) @@ -322,6 +347,6 @@ class TestCommandHandler implements CommandHandler { } } - return CommandResult.running() + return CommandResult.processing() } } diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateSerializationTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateSerializationTest.groovy index 23946b5c..2dcc9425 100644 --- a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateSerializationTest.groovy +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateSerializationTest.groovy @@ -146,7 +146,7 @@ class CommandStateSerializationTest extends Specification { decoded.result.deleted } - def 'should handle null result for running commands'() { + def 'should handle null result for processing commands'() { given: def encoder = new JacksonEncodingStrategy>() {} def now = Instant.now() @@ -154,7 +154,7 @@ class CommandStateSerializationTest extends Specification { def state = new TypedCommandState<>( 'cmd-xyz', 'create-job', - CommandStatus.RUNNING, + CommandStatus.PROCESSING, params, null, // No result yet null, @@ -171,7 +171,7 @@ class CommandStateSerializationTest extends Specification { decoded.params instanceof CreateJobParams decoded.params.image == 'ubuntu:22.04' decoded.result == null - decoded.status == CommandStatus.RUNNING + decoded.status == CommandStatus.PROCESSING } def 'should decode legacy JSON without error-tracking fields into the real record'() { @@ -198,11 +198,59 @@ class CommandStateSerializationTest extends Specification { then: 'existing fields survive' decoded.id() == 'cmd-legacy' - decoded.status() == CommandStatus.RUNNING + decoded.status() == CommandStatus.PROCESSING decoded.params() instanceof CreateJobParams decoded.params().image == 'alpine:latest' and: 'new fields default without a stored value — safe rolling deploy' decoded.errorsCount() == 0 decoded.modifiedAt() == null + and: 'a payload never carries a version — pre-versioning entries read as 0 (see VersionAware)' + decoded.version() == 0 + } + + def 'should decode the legacy status names written before the WorkQueue rename'() { + given: '''the encoder as wired by CommandStateStoreFactory, and state persisted by a replica + that still called the statuses SUBMITTED and RUNNING — during a rolling deploy, or + before it, for as long as the stored state lives (state-ttl)''' + def encoder = new JacksonEncodingStrategy() {} + def json = """\ + { + "id": "cmd-wire", + "type": "create-job", + "status": "${legacy}", + "params": null, + "result": null, + "error": null, + "errorsCount": 0, + "createdAt": "${Instant.now()}", + "startedAt": null, + "modifiedAt": null, + "completedAt": null + }""".stripIndent() + + when: + def decoded = encoder.decode(json) + + then: 'the @JsonAlias mapping carries it onto the current name — removing it strands the entry' + decoded.status() == current + + where: + legacy || current + 'SUBMITTED' || CommandStatus.PENDING + 'RUNNING' || CommandStatus.PROCESSING + } + + def 'version should never leak into the serialized payload'() { + given: 'the encoder as wired by CommandStateStoreFactory, and a state carrying a version' + def encoder = new JacksonEncodingStrategy() {} + def state = CommandState.create('cmd-ver', 'test', null).withVersion(42) + + when: + def json = encoder.encode(state) + + then: 'the version travels in the store frame, not the payload — the frame is the single source of truth' + !json.contains('"version"') + and: 'decoding yields version 0 until the store injects the frame version' + encoder.decode(json).version() == 0 } } diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateStoreRedisTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateStoreRedisTest.groovy new file mode 100644 index 00000000..2084d664 --- /dev/null +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateStoreRedisTest.groovy @@ -0,0 +1,165 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.command.store + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.UnaryOperator + +import com.github.f4b6a3.tsid.TsidCreator +import io.micronaut.context.ApplicationContext +import io.seqera.data.command.CommandConfig +import io.seqera.data.command.CommandState +import io.seqera.data.store.state.impl.RedisStateProvider +import io.seqera.data.store.state.impl.StateProvider +import io.seqera.fixtures.redis.RedisTestContainer +import io.seqera.serde.encode.StringEncodingStrategy +import io.seqera.serde.jackson.JacksonEncodingStrategy +import spock.lang.Shared +import spock.lang.Specification +/** + * The cross-replica half of the compare-and-swap proof. {@code CommandStateStoreUpdateTest} + * exercises the same logic against {@code LocalStateProvider}, which is in-JVM — but a single + * replica's dispatcher is single-threaded, so the realistic competing writer is a second + * replica reclaiming a message after the claim timeout, and the atomicity it relies on is + * the versioned {@code replaceIf} Lua script (libseqera#107), which the in-memory provider only + * approximates. This suite runs the load-bearing scenario through {@code RedisStateProvider} + * against a real Redis, with two store instances sharing it the way two replicas do — including + * replicas whose serialized forms differ byte-wise, the cross-pod condition that broke the + * byte-equality CAS this store used before (sched PR #913 rollout). + * + * @author Paolo Di Tommaso + */ +class CommandStateStoreRedisTest extends Specification implements RedisTestContainer { + + @Shared + ApplicationContext context + + def setup() { + context = ApplicationContext.run('test', 'redis') + sleep(500) // workaround to wait for Redis connection, as in RedisStateProviderTest upstream + } + + def cleanup() { + context.stop() + } + + /** + * Resolved by concrete class, the upstream convention ({@code RedisStateProviderTest}): both + * providers are active beans under the redis env, and asking for the concrete type either + * returns the Redis one or fails loudly — no silent fall-back to the in-memory provider, + * which is the failure mode this suite exists to close. + */ + private StateProvider redisProvider() { + return context.getBean(RedisStateProvider) + } + + private CommandStateStoreImpl replicaOn(StateProvider provider) { + return replicaOn(provider, new JacksonEncodingStrategy() {}) + } + + private CommandStateStoreImpl replicaOn(StateProvider provider, StringEncodingStrategy encoder) { + return new CommandStateStoreImpl(provider, encoder, context.getBean(CommandConfig)) + } + + /** + * Byte-divergent but JSON-equivalent encoding — the single-JVM stand-in for a replica whose + * JVM serializes the same value with a different property order (Jackson discovers derived + * accessors via {@code Class.getDeclaredMethods()}, whose order is process-dependent — the + * sched PR #913 incident). Any CAS comparing re-serialized bytes refuses such a replica's + * writes forever; the versioned CAS must not care. + */ + private StringEncodingStrategy divergentEncoder() { + final inner = new JacksonEncodingStrategy() {} + return new StringEncodingStrategy() { + @Override + String encode(CommandState value) { + final json = inner.encode(value) + return '{ ' + json.substring(1) + } + @Override + CommandState decode(String encoded) { + return inner.decode(encoded) + } + } + } + + def 'concurrent replicas should not lose a write nor spuriously fail'() { + given: 'two store instances sharing one Redis, as two replicas do' + def provider = redisProvider() + def replicaA = replicaOn(provider) + def replicaB = replicaOn(provider) + and: + def state = CommandState.create(TsidCreator.getTsid().toLowerCase(), 'test', null) + replicaA.save(state) + and: 'both writers collide inside the read-modify-write window; a CAS miss retries internally' + def start = new CountDownLatch(1) + def done = new CountDownLatch(2) + def applied = new AtomicInteger() + [replicaA, replicaB].eachWithIndex { store, i -> + Thread.start { + start.await(5, TimeUnit.SECONDS) + if (store.update(state.id(), { CommandState s -> + sleep(250) + s.withError("boom-$i") + } as UnaryOperator)) { + applied.incrementAndGet() + } + done.countDown() + } + } + + when: + start.countDown() + done.await(30, TimeUnit.SECONDS) + + then: 'both replicas succeeded first-call — contention is absorbed, never surfaced' + applied.get() == 2 + and: 'both increments survived across instances; a lost write would leave the count at 1' + replicaA.findById(state.id()).get().errorsCount() == 2 + } + + def 'replicas with byte-divergent serialization should still compare-and-swap'() { + given: 'two replicas whose encoders produce different bytes for the same value — the cross-pod condition of the sched PR #913 incident' + def provider = redisProvider() + def replicaA = replicaOn(provider) + def replicaB = replicaOn(provider, divergentEncoder()) + and: 'an entry written by replica A' + def state = CommandState.create(TsidCreator.getTsid().toLowerCase(), 'test', null) + replicaA.save(state) + + when: 'replica B transitions an entry it did not write, then A transitions B\'s write' + def appliedB = replicaB.update(state.id(), { CommandState s -> s.withError('from-b') } as UnaryOperator) + def appliedA = replicaA.update(state.id(), { CommandState s -> s.toProcessing() } as UnaryOperator) + + then: 'both cross-replica writes landed — a byte-comparing CAS would refuse them through every retry' + appliedB + appliedA + and: 'both transitions survived, whoever reads' + with(replicaA.findById(state.id()).get()) { + startedAt() != null + error() == 'from-b' + } + with(replicaB.findById(state.id()).get()) { + startedAt() != null + error() == 'from-b' + } + } + +} diff --git a/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateStoreUpdateTest.groovy b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateStoreUpdateTest.groovy new file mode 100644 index 00000000..724989be --- /dev/null +++ b/lib-cmd-queue-redis/src/test/groovy/io/seqera/data/command/store/CommandStateStoreUpdateTest.groovy @@ -0,0 +1,216 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.command.store + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.UnaryOperator + +import com.github.f4b6a3.tsid.TsidCreator +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.data.command.CommandConfig +import io.seqera.data.command.CommandState +import io.seqera.data.command.CommandStatus +import io.seqera.data.store.state.impl.StateProvider +import io.seqera.serde.jackson.JacksonEncodingStrategy +import jakarta.inject.Inject +import spock.lang.Specification +/** + * Covers {@code update()}: a command transition is a read-modify-write, and {@code save()} writes + * the whole record unconditionally, so two writers sharing a stale snapshot silently discard one + * of the two outcomes. + * + * @author Paolo Di Tommaso + */ +@MicronautTest(packages = ['io.seqera.data.workqueue'], transactional = false) +class CommandStateStoreUpdateTest extends Specification { + + @Inject + CommandStateStore store + + @Inject + StateProvider provider + + /** A store with an explicit update-attempts bound, everything else defaulted. */ + private CommandStateStoreImpl storeWithUpdateAttempts(int attempts) { + final config = new CommandConfig() { + @Override + int stateUpdateAttempts() { attempts } + } + return new CommandStateStoreImpl(provider, new JacksonEncodingStrategy() {}, config) + } + + private CommandState submitted() { + final state = CommandState.create(TsidCreator.getTsid().toLowerCase(), 'test', null) + store.save(state) + return state + } + + def 'concurrent transitions should not lose a write nor spuriously fail'() { + given: 'the load-bearing case — fails against a blind save() (lost write) and against a write-mutex (spurious refusal under contention)' + def state = submitted() + and: 'two writers that both read the same snapshot before either writes' + def start = new CountDownLatch(1) + def done = new CountDownLatch(2) + def applied = new AtomicInteger() + and: 'a slow mutator, so the window collides reliably instead of by scheduling luck' + 2.times { i -> + Thread.start { + start.await(5, TimeUnit.SECONDS) + // No caller-side retry: a CAS miss re-reads and re-applies INSIDE update(), + // so a live command must never see a refusal from mere contention. The 250ms + // hold would exhaust a lock-wait budget; the CAS just loses one round and wins + // the next. + if (store.update(state.id(), { CommandState s -> + sleep(250) + s.withError("boom-$i") + } as UnaryOperator)) { + applied.incrementAndGet() + } + done.countDown() + } + } + + when: + start.countDown() + done.await(30, TimeUnit.SECONDS) + + then: 'both writers succeeded first-call — contention is absorbed, never surfaced' + applied.get() == 2 + and: 'both increments survived; a lost write would leave the count at 1' + store.findById(state.id()).get().errorsCount() == 2 + } + + def 'update should apply the mutator to the stored state and persist it'() { + given: + def state = submitted() + + when: + def applied = store.update(state.id(), { CommandState s -> s.toProcessing() } as UnaryOperator) + + then: + applied + store.findById(state.id()).get().startedAt() != null + } + + def 'update should apply the mutator to the current value, not a stale snapshot'() { + given: 'a snapshot the caller holds, then a transition by someone else' + def stale = submitted() + store.update(stale.id(), { CommandState s -> s.withError('first') } as UnaryOperator) + + when: 'the caller mutates using its stale reference' + store.update(stale.id(), { CommandState s -> s.withError('second') } as UnaryOperator) + + then: 'the earlier error was not discarded — the streak accumulated' + def result = store.findById(stale.id()).get() + result.errorsCount() == 2 + result.error() == 'second' + } + + def 'update should pin the CAS witness to its own read, whatever the mutator did to the version'() { + given: 'a live command' + def state = submitted() + + when: 'a mutator hands back a state whose version is not the one the loop read' + def applied = store.update(state.id(), { CommandState s -> s.toProcessing().withVersion(0) } as UnaryOperator) + + then: 'the write still lands — an unpinned witness would miss every attempt and refuse silently' + applied + store.findById(state.id()).get().startedAt() != null + } + + def 'terminal result should not overwrite a cancellation'() { + given: + def state = submitted() + store.update(state.id(), { CommandState s -> s.cancelled() } as UnaryOperator) + + when: + def applied = store.update(state.id(), { CommandState s -> s.completed('done') } as UnaryOperator) + + then: + !applied + store.findById(state.id()).get().status() == CommandStatus.CANCELLED + } + + def 'cancellation should not overwrite a terminal result'() { + given: + def state = submitted() + store.update(state.id(), { CommandState s -> s.completed('done') } as UnaryOperator) + + when: + def applied = store.update(state.id(), { CommandState s -> s.cancelled() } as UnaryOperator) + + then: + !applied + def result = store.findById(state.id()).get() + result.status() == CommandStatus.SUCCEEDED + result.result() == 'done' + } + + def 'update should report false for an unknown command'() { + expect: + !store.update('cmd-does-not-exist', { CommandState s -> s.toProcessing() } as UnaryOperator) + } + + def 'update should give up after the configured update attempts'() { + given: 'a store bounded to 3 attempts and a mutator that always invalidates its own CAS' + def bounded = storeWithUpdateAttempts(3) + def state = CommandState.create(TsidCreator.getTsid().toLowerCase(), 'test', null) + bounded.save(state) + def invocations = new AtomicInteger() + + when: + def applied = bounded.update(state.id(), { CommandState s -> + invocations.incrementAndGet() + bounded.save(s.withError('concurrent')) // out-of-band write: the CAS always misses + s.toProcessing() + } as UnaryOperator) + + then: 'the loop ran exactly the configured number of rounds, then reported not applied' + !applied + invocations.get() == 3 + } + + def 'a non-positive update attempts bound should fail fast at construction'() { + when: + storeWithUpdateAttempts(attempts) + + then: + thrown(IllegalStateException) + + where: + attempts << [0, -1] + } + + def 'update should propagate a mutator failure and remain usable'() { + given: + def state = submitted() + + when: + store.update(state.id(), { CommandState s -> throw new RuntimeException('boom') } as UnaryOperator) + + then: + thrown(RuntimeException) + + and: 'a following transition proceeds normally — nothing is left behind to expire' + store.update(state.id(), { CommandState s -> s.toProcessing() } as UnaryOperator) + store.findById(state.id()).get().startedAt() != null + } + +} diff --git a/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/StallingCommandQueue.java b/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/StallingCommandQueue.java new file mode 100644 index 00000000..392e2ef6 --- /dev/null +++ b/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/StallingCommandQueue.java @@ -0,0 +1,112 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package io.seqera.data.command; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Replaces; +import io.micronaut.context.annotation.Requires; +import io.seqera.data.workqueue.WorkQueue; +import jakarta.inject.Singleton; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A command queue whose dispatcher never quiesces: {@code awaitQuiescent} spends every + * millisecond it is given and then reports failure — the {@code dispatcherStopped=false} + * case seen in production (#955). Used to make the drain's budget split observable. + * + *

Not a Spock mock: {@code @MockBean} wraps the bean in a proxy that cannot implement + * {@link CommandQueue}'s protected abstract methods, so the replacement is a real subclass + * installed through the factory below. + */ +class StallingCommandQueue extends CommandQueue { + + private static final Logger log = LoggerFactory.getLogger(StallingCommandQueue.class); + + /** + * Budget handed to the FIRST {@code awaitQuiescent} call — the one {@code drain()} makes at + * step 1, which is the split under test. {@code drain()} calls it again through + * {@code close()} at step 3, and that later call describes the leftover instead. + */ + static final AtomicLong FIRST_QUIESCE_BUDGET_MILLIS = new AtomicLong(-1); + + /** + * When set, calls after the first report that the dispatcher has stopped — the slow-but-not-stuck + * dispatcher, which outlives step 1's capped wait but stops while {@code close()} waits with the + * leftover at step 3. Left unset, no call ever reports it stopped, which is the stuck dispatcher + * seen in production (#955). + */ + static final AtomicBoolean STOPS_DURING_CLOSE = new AtomicBoolean(false); + + /** + * Only the first call sleeps. Every call reports failure — the dispatcher never stops — but + * sleeping again inside {@code close()} at step 3 would spend the leftover budget too, making + * the drain take the full timeout whatever step 1 did and hiding the very difference this + * exists to expose. The real implementation returns immediately once the thread has stopped. + */ + private final AtomicBoolean stalled = new AtomicBoolean(false); + + StallingCommandQueue(WorkQueue target) { + super(target); + } + + @Override + protected String name() { + return "stalling-command-queue"; + } + + @Override + protected Duration pollInterval() { + return Duration.ofMillis(100); + } + + @Override + public boolean awaitQuiescent(Duration timeout) { + FIRST_QUIESCE_BUDGET_MILLIS.compareAndSet(-1, timeout.toMillis()); + if (stalled.compareAndSet(false, true)) { + try { + Thread.sleep(Math.max(0, timeout.toMillis())); + } + catch (InterruptedException e) { + log.info("Stalling command queue interrupted while spending its quiesce budget", e); + Thread.currentThread().interrupt(); + } + return false; + } + return STOPS_DURING_CLOSE.get(); + } +} + +/** + * Installs {@link StallingCommandQueue} in place of the default test queue, for the one spec + * that needs a dispatcher which never stops. Gated on a property so every other spec in this + * module keeps the normal {@code TestCommandQueue}. + */ +@Factory +@Requires(property = "test.command-queue.stalling", value = "true") +class StallingCommandQueueFactory { + + @Singleton + @Replaces(bean = CommandQueue.class, factory = TestCommandQueueFactory.class) + CommandQueue commandQueue(WorkQueue target) { + return new StallingCommandQueue(target); + } +} diff --git a/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/TestCommandConfig.java b/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/TestCommandConfig.java index 0e293cf8..f8d40863 100644 --- a/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/TestCommandConfig.java +++ b/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/TestCommandConfig.java @@ -32,20 +32,29 @@ public class TestCommandConfig implements CommandConfig { @Value("${command.poll-interval:100ms}") private Duration pollInterval; - @Value("${command.execute-timeout:1s}") - private Duration executeTimeout; + @Value("${command.max-concurrency:25}") + private int maxConcurrency; @Value("${command.state.ttl:1h}") private Duration stateTtl; + /** Short default so RUNNING re-polls stay fast in tests (prod default is 45s). */ + @Value("${command.check-status-interval:300ms}") + private Duration checkStatusInterval; + @Override public Duration pollInterval() { return pollInterval; } @Override - public Duration executeTimeout() { - return executeTimeout; + public Duration checkStatusInterval() { + return checkStatusInterval; + } + + @Override + public int maxConcurrency() { + return maxConcurrency; } @Override diff --git a/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/TestCommandQueue.java b/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/TestCommandQueue.java index 28c08010..7b8d5c7a 100644 --- a/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/TestCommandQueue.java +++ b/lib-cmd-queue-redis/src/test/java/io/seqera/data/command/TestCommandQueue.java @@ -19,7 +19,7 @@ import java.time.Duration; import io.micronaut.context.annotation.Factory; -import io.seqera.data.stream.MessageStream; +import io.seqera.data.workqueue.WorkQueue; import jakarta.inject.Singleton; /** @@ -27,7 +27,7 @@ */ class TestCommandQueue extends CommandQueue { - TestCommandQueue(MessageStream target) { + TestCommandQueue(WorkQueue target) { super(target); } @@ -49,7 +49,7 @@ protected Duration pollInterval() { class TestCommandQueueFactory { @Singleton - CommandQueue commandQueue(MessageStream target) { + CommandQueue commandQueue(WorkQueue target) { return new TestCommandQueue(target); } } diff --git a/lib-cmd-queue-redis/src/test/resources/application-test.yml b/lib-cmd-queue-redis/src/test/resources/application-test.yml index 72717358..57bc440d 100644 --- a/lib-cmd-queue-redis/src/test/resources/application-test.yml +++ b/lib-cmd-queue-redis/src/test/resources/application-test.yml @@ -7,13 +7,17 @@ micronaut: # Command queue configuration command: poll-interval: 100ms - execute-timeout: 1s + max-concurrency: 25 state: ttl: 1h command-queue: poll-interval: 100ms - stream-name: "command-queue-test/v1" checker: enabled: true interval: 500ms + +# Local queue retry pacing: keep redelivery cycles fast in tests (prod default is 1s) +workqueue: + local: + retry-delay: 100ms diff --git a/lib-cmd-queue-redis/src/test/resources/logback-test.xml b/lib-cmd-queue-redis/src/test/resources/logback-test.xml index 2140c7df..1eec64a0 100644 --- a/lib-cmd-queue-redis/src/test/resources/logback-test.xml +++ b/lib-cmd-queue-redis/src/test/resources/logback-test.xml @@ -24,7 +24,7 @@ - + diff --git a/lib-data-stream-redis/README.md b/lib-data-stream-redis/README.md index d77925b5..51c267be 100644 --- a/lib-data-stream-redis/README.md +++ b/lib-data-stream-redis/README.md @@ -2,6 +2,18 @@ Message streaming with Redis Streams and local implementations for persistent event processing. +> **Superseded by [`lib-data-workqueue`](../lib-data-workqueue/README.md)** (+ +> [`lib-data-workqueue-redis`](../lib-data-workqueue-redis/README.md)), which carries the same +> abstraction forward under names that match what it actually implements — a reliable work +> queue with competing consumers and a message lease — and adds lease settlement, heartbeat +> renewal and a cooperative drain. +> +> This module stays published and supported at 1.5.0 for the services that pin it; it has no +> in-repo consumer of its own. New consumers should start on `lib-data-workqueue`. Migrating is +> not transparent: the SPI differs, and the metric names and tags changed +> (`seqera.stream.*` → `seqera.workqueue.*`, tag `stream` → `queue`, `stream_id` → `queue_id`), +> so dashboards and alerts have to be updated with the swap. + ## Installation Add this dependency to your `build.gradle`: diff --git a/lib-data-workqueue-redis/README.md b/lib-data-workqueue-redis/README.md index 0b6728d5..973fd147 100644 --- a/lib-data-workqueue-redis/README.md +++ b/lib-data-workqueue-redis/README.md @@ -1,59 +1,96 @@ # lib-data-workqueue-redis -The Redis-backed implementation of the [`lib-data-workqueue`](../lib-data-workqueue) -abstraction. It implements a distributed, reliable work queue on top of Redis Streams -consumer groups: competing consumers, one live owner per key, acknowledgment, a -visibility-timeout lease kept alive by heartbeat renewal, redelivery and dead-owner reclaim. +Redis backend of the work queue: the Redis Streams implementation of the `WorkQueue` SPI +declared by [`lib-data-workqueue`](../lib-data-workqueue/README.md), including the message +lease (a heartbeat on the pending-entries-list entry) that lets a live handler run past the +visibility timeout without its entry being stolen. -> **Migrating from `lib-data-stream-redis`?** This module (together with -> `lib-data-workqueue`) is the split/rename of `lib-data-stream-redis` 1.6.0. It keeps the -> exact behaviour (the Redis lease renewal still uses `XCLAIM … JUSTID` internally) and only -> renames the abstraction. See the full guide at -> [`docs/superpowers/specs/2026-07-11-workqueue-rename-migration.md`](../docs/superpowers/specs/2026-07-11-workqueue-rename-migration.md). +> **Supersedes the Redis half of `lib-data-stream-redis`.** This module is the Redis Streams +> backend of [`lib-data-workqueue`](../lib-data-workqueue/README.md), and like its core it +> fills the placeholder that [#86](https://github.com/seqeralabs/libseqera/pull/86) named and +> [#100](https://github.com/seqeralabs/libseqera/pull/100) left unpublished. +> +> **2.0.0 is a breaking change over the unpublished 1.0.0** — it implements the +> `consume()`/`Decision`/`MessageLease` SPI rather than the `receive()`/`Lease` one, and +> owns the lease-renewal daemon that used to live in `AbstractWorkQueue`. Versioned in +> lockstep with `lib-data-workqueue`; the two must be upgraded together. +> +> The published `io.seqera:lib-data-stream-redis` artifact still exists and is still used by +> other services on older pinned versions; those are unaffected by anything here. ## Installation -Add these dependencies to your `build.gradle`: +It exposes +`lib-data-workqueue` as an `api` dependency, since `RedisWorkQueue` publicly implements +`WorkQueue`: ```gradle dependencies { - implementation 'io.seqera:lib-data-workqueue:1.0.0' - implementation 'io.seqera:lib-data-workqueue-redis:1.0.0' + implementation 'io.seqera:lib-data-workqueue-redis:2.0.0' } ``` -## What's here +`RedisWorkQueue` is a `@Singleton` gated on `@Requires(bean = RedisActivator.class)`, so it +activates only when the `redis` environment is active — otherwise `LocalWorkQueue` from the core +module takes its place, with no code change in the consumer. -| Type | Description | -|---|---| -| `RedisWorkQueue` | `implements WorkQueue`; distributed queue over Redis Streams consumer groups. Auto-activated when the `redis` environment is active (requires a configured `JedisPool`). | -| `RedisWorkQueueConfig` | Configuration seam: `getDefaultConsumerGroupName()`, `getVisibilityTimeout()`, `getConsumerWarnTimeout()`, `getHeartbeatInterval()` (default `visibility-timeout / 3`), `getMaxProcessingTime()` (default `15m`), plus the `*Millis()` variants. | +## Delivery model -The API/SPI (`WorkQueue`, `WorkQueue.Lease`), the abstract base (`AbstractWorkQueue`), the -in-memory `LocalWorkQueue`, `MessageConsumer` and the metrics seam all live in -[`lib-data-workqueue`](../lib-data-workqueue) — see its README for the full architecture, -metrics and configuration reference. +One dispatcher poll does, in order: + +1. `XAUTOCLAIM` — take over any **stalled** entry whose owner has not renewed within the + visibility timeout (a dead consumer). +2. `XREADGROUP >` — otherwise read a new entry for this consumer group. +3. Register the entry in the in-flight **lease** registry, keyed per queue (entry IDs are unique + only per Redis stream key, so a bare `StreamEntryID` key would collide across queues). +4. Run the consumer and settle on its `Decision`: `ACK` → `XACK` + `XDEL`; `RETRY` → drop the + lease so the idle clock resumes and the entry redelivers; `DEFERRED` → leave the entry leased + until the consumer's task settles it through `MessageLease`. + +A background daemon scheduler renews every in-flight entry **per queue in one round-trip** at +`visibility timeout / 4`: a pipelined per-id `XPENDING` ownership check followed by one variadic +`XCLAIM JUSTID`. An entry taken over during a renewal outage is dropped rather than re-seized +(no ownership ping-pong) and counted on `seqera.workqueue.lease.lost`; a lease older than the max +lease age whose owner is not provably alive — `MessageLease.bindLiveness()` — stops being renewed +so the claim cycle recovers it, counted on `seqera.workqueue.lease.leak`. + +The full design, including the renewal margin math and the residual duplicate windows, is in +[`docs/plans/command-execution-guarantee-message-lease.md`](../docs/plans/command-execution-guarantee-message-lease.md). ## Configuration -Provide a `RedisWorkQueueConfig` bean; the `visibility-timeout` property governs the -dead-consumer failover window (mapped to the Redis consumer-group min-idle used by -`XAUTOCLAIM`). - -```groovy -@Requires(env = 'redis') -@Singleton -class MyRedisWorkQueueConfig implements RedisWorkQueueConfig { - @Override String getDefaultConsumerGroupName() { 'my-service-group' } - @Override Duration getVisibilityTimeout() { Duration.ofSeconds(60) } - @Override Duration getConsumerWarnTimeout() { Duration.ofSeconds(5) } -} -``` +`RedisWorkQueueConfig` is the SPI the consumer implements — this module binds no property +keys of its own. The consumer chooses the prefix; the settings below, with the values the +Seqera scheduler runs, are the reference for what each one governs: + +| Key | Default | Meaning | +|---|---|---| +| `.consumer-group` | `sched-workers` | Redis consumer-group name shared by all replicas. | +| `.visibility-timeout` | `20s` | Idle time after which an unrenewed entry is claimable by another replica — dead-consumer detection, and the transient-error retry cadence. | +| `.consumer-warn-timeout` | `15s` | Warn when a synchronous consume cycle exceeds this. | +| `.lease-renewal-period` | derived: `visibility-timeout / 4` | Renewal tick. Startup fails on a period at or above the visibility timeout. | +| `.max-lease-age` | derived: `3 × visibility-timeout` | Leak backstop for an unsettled lease with no provably-live owner. | + +The Redis connection itself comes from `lib-jedis-pool` (`redis.uri`, `redis.pool.*`). Note +`redis.pool.maxWait`: an unbounded borrow inside a renewal tick would block the single-threaded +scheduler and silently let every lease on the replica go stalled — set it below the renewal +period so a starved tick fails loudly instead. + +## Metrics + +Recorded through the optional `QueueMetrics` seam of `lib-data-workqueue` — see +[that module's README](../lib-data-workqueue/README.md#metrics-optional) for the meter table. +`RedisWorkQueue` injects `QueueMetrics` as an *optional* bean and falls back to a no-op, so a +deployment that wants the lease meters must provide the bean, typically from the same factory +that builds the queue and guarded on a `MeterRegistry` being present. ## Testing -Uses Testcontainers to spin up a real Redis, so a running Docker daemon is required. +The tests run against a real Redis in a Testcontainer, so **Docker must be available**: ```bash ./gradlew :lib-data-workqueue-redis:test ``` + +`RedisWorkQueueLeaseTest` is the suite that would catch an accidental change to the lease +semantics — renewal margin, ownership check, age backstop, settlement idempotence. diff --git a/lib-data-workqueue-redis/VERSION b/lib-data-workqueue-redis/VERSION index 3eefcb9d..227cea21 100644 --- a/lib-data-workqueue-redis/VERSION +++ b/lib-data-workqueue-redis/VERSION @@ -1 +1 @@ -1.0.0 +2.0.0 diff --git a/lib-data-workqueue-redis/build.gradle b/lib-data-workqueue-redis/build.gradle index 5ded163c..c14d01fd 100644 --- a/lib-data-workqueue-redis/build.gradle +++ b/lib-data-workqueue-redis/build.gradle @@ -41,7 +41,6 @@ dependencies { testImplementation "org.apache.groovy:groovy-nio:4.0.31" testImplementation "org.apache.groovy:groovy-templates:4.0.31" testImplementation "org.apache.groovy:groovy-json:4.0.31" - testImplementation project(':lib-lang') testImplementation project(':lib-random') testImplementation project(':lib-serde') testImplementation project(':lib-retry') diff --git a/lib-data-workqueue-redis/changelog.txt b/lib-data-workqueue-redis/changelog.txt index 56bbd7a0..2bc9ca1e 100644 --- a/lib-data-workqueue-redis/changelog.txt +++ b/lib-data-workqueue-redis/changelog.txt @@ -1,5 +1,19 @@ # lib-data-workqueue-redis changelog +2.0.0 - 19 Aug 2026 +- BREAKING change over the unpublished 1.0.0, versioned in lockstep with lib-data-workqueue + 2.0.0. The two must be upgraded together. +- Implements the consume()/Decision/MessageLease SPI in place of receive()/Lease. +- Owns the lease-renewal daemon that used to live in AbstractWorkQueue: every in-flight entry + is renewed per queue in one round-trip, gated on the liveness supplier bound to its lease, so + an entry whose owner is provably alive is never reclaimed while a dead owner's entry is. +- RedisWorkQueueConfig gains lease-renewal-period and max-lease-age (both derived from + visibility-timeout by default). Startup fails on a renewal period at or above the visibility + timeout. +- Tests move from package io.seqera.data.workqueue to io.seqera.data.workqueue.redis, matching + the main sources, and add lease and config coverage. +- Dropped the lib-lang test dependency; no test imports io.seqera.lang. + 1.0.0 - 11 Jul 2026 - Initial release. This module is the Redis split of lib-data-stream-redis 1.6.0, renamed to reflect the reliable work-queue semantics it implements on top of Redis Streams diff --git a/lib-data-workqueue-redis/src/main/java/io/seqera/data/workqueue/redis/RedisWorkQueue.java b/lib-data-workqueue-redis/src/main/java/io/seqera/data/workqueue/redis/RedisWorkQueue.java index ba56d62c..015fbba4 100644 --- a/lib-data-workqueue-redis/src/main/java/io/seqera/data/workqueue/redis/RedisWorkQueue.java +++ b/lib-data-workqueue-redis/src/main/java/io/seqera/data/workqueue/redis/RedisWorkQueue.java @@ -18,32 +18,51 @@ package io.seqera.data.workqueue.redis; import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; import io.micronaut.context.annotation.Requires; +import io.micronaut.core.annotation.Nullable; import io.seqera.activator.redis.RedisActivator; import io.seqera.data.workqueue.MessageConsumer; +import io.seqera.data.workqueue.MessageLease; import io.seqera.data.workqueue.WorkQueue; +import io.seqera.data.workqueue.metrics.NoopQueueMetrics; +import io.seqera.data.workqueue.metrics.QueueMetrics; import io.seqera.random.LongRndKey; import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import jakarta.inject.Inject; import jakarta.inject.Singleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; import redis.clients.jedis.StreamEntryID; import redis.clients.jedis.exceptions.JedisDataException; import redis.clients.jedis.params.XAutoClaimParams; import redis.clients.jedis.params.XClaimParams; +import redis.clients.jedis.params.XPendingParams; import redis.clients.jedis.params.XReadGroupParams; import redis.clients.jedis.resps.StreamEntry; +import redis.clients.jedis.resps.StreamPendingEntry; /** * Redis-based implementation of {@link WorkQueue} that provides a distributed, reliable - * work queue using Redis Streams consumer groups as the underlying storage mechanism. + * work queue using Redis Streams as the underlying storage mechanism. * *

This implementation offers the following features: *

    @@ -51,25 +70,33 @@ *
  • Reliability: Guarantees message delivery consistency across service restarts
    • *
  • Consumer Groups: Uses Redis consumer groups for load balancing and fault tolerance
    • *
  • Message Claiming: Automatically reclaims stalled messages from failed consumers
    • + *
  • Message Leases: Entries handed to a consumer stay leased — a background + * heartbeat renews their idle clock so a live handler can run past the visibility + * timeout without the entry going stalled; the visibility timeout degrades to its + * one legitimate purpose, detecting a dead consumer
    • *
  • Persistence: Messages are persisted in Redis until explicitly acknowledged and deleted
    • *
* - *

The implementation follows Redis Streams best practices: - *

    - *
  • Creates consumer groups automatically on initialization
    • - *
  • Uses unique consumer names to avoid conflicts
    • - *
  • Implements message claiming for handling consumer failures
    • - *
  • Acknowledges and removes processed messages to prevent memory bloat
    • - *
- * *

Message processing workflow: *

    *
  1. Attempt to claim any stalled messages from failed consumers
    2. *
  2. If no stalled messages, read new messages from the queue
    3. + *
  3. Register the entry in the in-flight lease registry
    4. *
  4. Process the message through the provided consumer
    5. - *
  5. Acknowledge and delete the message upon successful processing
    6. + *
  6. Settle according to the consumer's {@link MessageConsumer.Decision}: {@code ACK} + * acknowledges and deletes the entry, {@code RETRY} releases the lease so the entry + * redelivers after the visibility timeout, {@code DEFERRED} leaves the entry leased + * until the consumer's task settles it via {@link MessageLease}
    7. *
* + *

Lease renewal: a single daemon scheduler renews all in-flight entries per queue in + * one round-trip at {@code visibility timeout / 4}. Renewal performs an ownership check + * first (an entry taken over by another consumer during a renewal outage is dropped, never + * re-seized) and applies a liveness-gated age backstop (a lease older than the max lease + * age whose owner is not provably alive — see {@link MessageLease#bindLiveness} — is a + * registry leak: renewal stops so the claim cycle can recover the entry; a lease whose + * bound owner is still running is never age-pruned). + * *

This class is automatically activated when the 'redis' environment is active * and requires a configured {@link JedisPool} for Redis connectivity. * @@ -92,6 +119,10 @@ public class RedisWorkQueue implements WorkQueue { @Inject private RedisWorkQueueConfig config; + @Inject + @Nullable + private QueueMetrics metrics; + private String consumerName; /** @@ -101,10 +132,79 @@ public class RedisWorkQueue implements WorkQueue { */ private final Map lastClaimCursor = new ConcurrentHashMap<>(); + /** + * Leased entries per queue: registered before the consumer runs, removed on settle. + * Queue-scoped (composite) keying — a bare {@link StreamEntryID} key would collide + * across queues, since entry IDs are unique only per Redis stream key. + */ + private final Map> inFlight = new ConcurrentHashMap<>(); + + private ScheduledExecutorService renewalScheduler; + + /** + * Completion timestamp (nanos) of the last renewal tick — the liveness signal behind + * the {@code lease.renewal.age} gauge. A renewal thread blocked before the tick's + * finally (e.g. on an unbounded pool borrow) is otherwise INVISIBLE: no exception, no + * renewError, no overrun warn — while every lease on the replica quietly ages toward + * the visibility timeout. Initialized at creation so the gauge measures scheduler + * silence from startup, not from the first successful tick. + */ + private volatile long lastTickCompletedNanos; + + private long renewPeriodNanos; + + private long maxLeaseAgeNanos; + @PostConstruct private void create() { consumerName = "consumer-" + LongRndKey.rndLong(); - log.info("Creating Redis work queue - consumer={}", consumerName); + if (metrics == null) { + metrics = NoopQueueMetrics.INSTANCE; + } + // Fail fast on tuning that cannot protect anything — never silently convert an + // invalid value into unsafe behavior (a clamped 1ms period would mean ~1000 + // renewal pipelines per second; a non-positive max age would prune every unbound + // lease on its first tick). Warn when the period eats the missed-tick tolerance + // (default visibility-timeout/4 tolerates two missed ticks). + final long renewPeriodMillis = config.getLeaseRenewalPeriodMillis(); + if (renewPeriodMillis <= 0) { + throw new IllegalStateException("Lease renewal period must be positive - offending value: " + renewPeriodMillis + "ms"); + } + if (renewPeriodMillis >= config.getVisibilityTimeoutMillis()) { + throw new IllegalStateException("Lease renewal period (" + renewPeriodMillis + + "ms) must be below the visibility timeout (" + config.getVisibilityTimeoutMillis() + "ms)"); + } + if (renewPeriodMillis > config.getVisibilityTimeoutMillis() / 3) { + log.warn("Lease renewal period {}ms leaves no missed-tick tolerance before the visibility timeout {}ms - a single failed renewal may lose leases", + renewPeriodMillis, config.getVisibilityTimeoutMillis()); + } + final long maxLeaseAgeMillis = config.getMaxLeaseAgeMillis(); + if (maxLeaseAgeMillis <= 0) { + throw new IllegalStateException("Max lease age must be positive - offending value: " + maxLeaseAgeMillis + "ms"); + } + if (maxLeaseAgeMillis < config.getVisibilityTimeoutMillis()) { + log.warn("Max lease age {}ms is below the visibility timeout {}ms - the leak backstop may prune unbound leases before their first claim window elapses", + maxLeaseAgeMillis, config.getVisibilityTimeoutMillis()); + } + renewPeriodNanos = TimeUnit.MILLISECONDS.toNanos(renewPeriodMillis); + maxLeaseAgeNanos = TimeUnit.MILLISECONDS.toNanos(maxLeaseAgeMillis); + renewalScheduler = Executors.newSingleThreadScheduledExecutor(task -> { + final Thread thread = new Thread(task, "redis-workqueue-lease-renewal"); + thread.setDaemon(true); + return thread; + }); + lastTickCompletedNanos = System.nanoTime(); + metrics.bindRenewalLiveness(this::renewalAgeNanos); + renewalScheduler.scheduleAtFixedRate(this::renewLeases, renewPeriodMillis, renewPeriodMillis, TimeUnit.MILLISECONDS); + log.info("Creating Redis work queue - consumer={}; lease renewal period={}ms; max lease age={}ms", + consumerName, renewPeriodMillis, config.getMaxLeaseAgeMillis()); + } + + @PreDestroy + private void destroy() { + if (renewalScheduler != null) { + renewalScheduler.shutdownNow(); + } } protected boolean initGroup0(Jedis jedis, String queueId, String group) { @@ -144,13 +244,16 @@ public void offer(String queueId, String message) { /** * {@inheritDoc} * - *

Reads one entry (a reclaimed stalled one via {@code XAUTOCLAIM}, otherwise a - * newly delivered one via {@code XREADGROUP >}) without acking it. - * The returned lease id is the Redis {@link StreamEntryID} of the delivered entry. + *

Registration bracket: the entry is registered in the lease registry before + * the consumer is invoked and un-registered on every path except a returned + * {@link MessageConsumer.Decision#DEFERRED} — including a thrown exception, which + * settles as {@code RETRY} and propagates to the caller. Only the {@code DEFERRED} + * return value transfers the lease to the consumer's task. */ @Override - public Lease receive(String queueId) { + public MessageConsumer.Decision consume(String queueId, MessageConsumer consumer) { try (Jedis jedis = pool.getResource()) { + final long begin = System.currentTimeMillis(); StreamEntry entry = claimMessage(jedis, queueId); if (entry == null) { entry = readMessage(jedis, queueId); @@ -158,103 +261,262 @@ public Lease receive(String queueId) { if (entry == null) { return null; } - return new Lease<>(entry.getID().toString(), entry.getFields().get(DATA_FIELD)); + final String msg = entry.getFields().get(DATA_FIELD); + final Lease lease = register(queueId, entry.getID()); + final MessageConsumer.Decision decision = acceptMessage(consumer, msg, lease, queueId); + settle(jedis, lease, decision, begin, msg); + return decision; } } /** - * {@inheritDoc} - * - *

Resets the idle time of the entry to zero by re-claiming it to this same - * consumer with a {@code min-idle} of {@code 0} using {@code XCLAIM … JUSTID}, - * so an alive consumer keeps ownership of the message regardless of how long the - * handler runs. + * Invoke the consumer under the registration bracket: a thrown exception (or a null + * decision) un-registers the entry — settling it as RETRY, redelivered after the + * visibility timeout — and propagates to the caller. */ - @Override - public void renewLease(String queueId, String leaseId) { - try (Jedis jedis = pool.getResource()) { - jedis.xclaimJustId( - queueId, - config.getDefaultConsumerGroupName(), - consumerName, - 0L, - XClaimParams.xClaimParams(), - new StreamEntryID(leaseId)); + private MessageConsumer.Decision acceptMessage(MessageConsumer consumer, String msg, Lease lease, String queueId) { + try { + return Objects.requireNonNull(consumer.accept(msg, lease), "Message consumer returned a null decision"); + } + catch (Throwable e) { + lease.settleAsRetry(); + log.error("Redis work queue - consumer errored for queue={}; entry={} - the entry will be redelivered after the visibility timeout", queueId, lease.entryId, e); + throw e; } } /** - * {@inheritDoc} - * - *

Acknowledges the entry ({@code XACK}) and permanently removes it from the - * queue ({@code XDEL}) atomically so it can neither be claimed nor redelivered. + * Settle a synchronous decision. {@code DEFERRED} leaves the entry registered — the + * consumer's task owns the lease and settles it via {@link MessageLease}. For + * {@code ACK} and {@code RETRY} the first settlement wins: when the consumer already + * settled through the lease, the returned decision is a no-op. */ - @Override - public void ack(String queueId, String leaseId) { - final var id = new StreamEntryID(leaseId); - try (Jedis jedis = pool.getResource()) { - final var tx = jedis.multi(); - // acknowledge the entry has been processed so that it cannot be claimed anymore - tx.xack(queueId, config.getDefaultConsumerGroupName(), id); - // this removes permanently the entry from the queue - tx.xdel(queueId, id); - tx.exec(); + private void settle(Jedis jedis, Lease lease, MessageConsumer.Decision decision, long begin, String msg) { + if (decision == MessageConsumer.Decision.DEFERRED) { + return; } + if (!lease.trySettle()) { + return; + } + unregister(lease); + if (decision == MessageConsumer.Decision.ACK) { + final long delta = System.currentTimeMillis() - begin; + if (delta > config.getConsumerWarnTimeoutMillis()) { + log.warn("Redis work queue - consume processing took {} - offending entry={}; message={}", + Duration.ofMillis(delta), lease.entryId, msg); + } + ackAndDelete(jedis, lease.queueId, lease.entryId); + } + // RETRY: stopping renewal is the release — the entry stays pending and the + // claim cadence is the retry schedule; no Redis call at all } /** - * {@inheritDoc} - * - *

No-op: the entry remains in the pending-entries list and becomes reclaimable - * by a peer consumer once its idle time exceeds the visibility timeout. + * Acknowledge the entry and permanently remove it from the queue, atomically. */ - @Override - public void release(String queueId, String leaseId) { - // no-op: entry stays in the PEL, reclaimable after the visibility timeout + private void ackAndDelete(Jedis jedis, String queueId, StreamEntryID entryId) { + final var tx = jedis.multi(); + tx.xack(queueId, config.getDefaultConsumerGroupName(), entryId); + tx.xdel(queueId, entryId); + tx.exec(); + } + + private Lease register(String queueId, StreamEntryID entryId) { + final Lease lease = new Lease(queueId, entryId); + inFlight.computeIfAbsent(queueId, k -> new ConcurrentHashMap<>()).put(entryId, lease); + return lease; } /** - * {@inheritDoc} - * - *

Derived from the configured visibility timeout so an alive consumer's lease - * is renewed well before a peer could reclaim it. + * Remove a lease from the in-flight registry. Removes by (key, value) so a stale + * lease for a re-claimed entry can never evict the registration of its successor. */ - @Override - public Duration heartbeatInterval() { - return config.getHeartbeatInterval(); + private void unregister(Lease lease) { + final Map leases = inFlight.get(lease.queueId); + if (leases != null) { + leases.remove(lease.entryId, lease); + } + } + + private long renewalAgeNanos() { + return System.nanoTime() - lastTickCompletedNanos; + } + + private int leasedCount() { + int count = 0; + for (Map leases : inFlight.values()) { + count += leases.size(); + } + return count; } /** - * {@inheritDoc} + * Age of the oldest currently-leased entry, in nanoseconds; zero when nothing is + * leased. Backs the max-lease-age gauge sampled on every renewal tick. */ - @Override - public Duration maxProcessingTime() { - return config.getMaxProcessingTime(); + private long oldestLeaseAgeNanos() { + long max = 0; + for (Map leases : inFlight.values()) { + for (Lease lease : leases.values()) { + max = Math.max(max, lease.ageNanos()); + } + } + return max; } /** - * {@inheritDoc} + * Renew all in-flight leases. Scheduled at {@code fixedRate = visibilityTimeout / 4}. + * The tick body catches {@link Throwable} — an escaping {@link Error} would silently + * cancel a fixedRate task forever, losing every lease on the replica at once. */ - @Override - public boolean consume(String queueId, MessageConsumer consumer) { - final long begin = System.currentTimeMillis(); - final Lease lease = receive(queueId); - if (lease == null) { - return false; - } - if (consumer.accept(lease.message())) { - ack(queueId, lease.id()); - final var delta = System.currentTimeMillis() - begin; - if (delta > config.getConsumerWarnTimeoutMillis()) { - log.warn("Redis work queue - consume processing took {} - offending entry={}; message={}", - Duration.ofMillis(delta), lease.id(), lease.message()); + void renewLeases() { + final long start = System.nanoTime(); + try { + inFlight.forEach(this::renewQueue); + } + catch (Throwable t) { + log.error("Lease renewal tick failed", t); // outer wall; renewQueue contains per queue + } + finally { + final long elapsed = System.nanoTime() - start; + lastTickCompletedNanos = System.nanoTime(); + metrics.renewTick(elapsed, leasedCount(), oldestLeaseAgeNanos()); + if (elapsed > renewPeriodNanos) { + log.warn("Lease renewal tick took {} - exceeds the renewal period; leases at risk", Duration.ofNanos(elapsed)); } - return true; } - else { - release(queueId, lease.id()); - return false; + } + + private void renewQueue(String queueId, Map leases) { + if (leases.isEmpty()) { + return; + } + /* One immutable snapshot drives the WHOLE tick — query, backstop, prune and renew. + Iterating the live map after the ownership query would race the dispatcher: a + lease registered while the pipeline was in flight was never queried, and treating + its missing owner as "no longer pending" would drop a newborn lease (and its + heartbeat) — resurfacing the take-over-mid-flight bug. Entries registered after + the snapshot are renewed on the NEXT tick, which is safe by construction: a + newborn was just claimed or read (idle clock ≈ 0) and the renewal period is + validated to be well below the visibility timeout. */ + final List> snapshot = List.copyOf(leases.entrySet()); + try (Jedis jedis = pool.getResource()) { + /* 1. Ownership check, one XPENDING range call: an entry that went stalled during + a renewal outage now belongs to another consumer. Renewing it blindly + (minIdle=0 seizes regardless of owner) would take it back mid-execution — + ownership ping-pong. Instead: drop it, count it, log it. This makes the + residual duplicate window OBSERVABLE instead of silent. + 2. Age backstop, liveness-gated: a lease older than the max lease age whose + owner is not provably alive is a registry leak (a settlement path that + never ran) — stop renewing, log loudly, let the claim cycle recover the + entry. A leak can never be permanent, and a live handler is never + age-pruned, however long it runs. + 3. One variadic XCLAIM JUSTID for everything still ours — a single round-trip + per queue per tick, so tick duration does not scale with in-flight count + (25 sequential renewals against a slow-but-alive Redis would exceed the + visibility timeout and lose every lease exactly when Redis degrades). */ + final List mine = checkOwnershipAndPrune(jedis, queueId, leases, snapshot); // XPENDING + if (!mine.isEmpty()) { + jedis.xclaimJustId(queueId, config.getDefaultConsumerGroupName(), consumerName, + 0, new XClaimParams(), mine.toArray(StreamEntryID[]::new)); + } + } + catch (Exception e) { + // Transient: the next tick retries — two consecutive failed ticks are tolerated + // before a lease is at risk (period = visibility timeout / 4). + metrics.renewError(); + log.warn("Lease renewal errored for queue {}, will retry", queueId, e); + } + } + + /** + * Return the leased entry ids still owned by this consumer, pruning from the registry + * the leases lost to another consumer, the leases whose entry is no longer pending at + * all, and those older than the age backstop whose owner is not provably alive. + * + *

Two invariants: an id is renewed only when its ownership was positively + * confirmed this tick — unknown must never degrade to "renew", because the + * renewal XCLAIM (minIdle=0) seizes regardless of the current owner — and only + * the ids in the snapshot are ever judged: a lease registered concurrently was + * never queried, so judging it against this tick's answers would misread it as + * "no longer pending" and drop it. Removals go through the live map conditionally + * ({@code remove(key, value)}), so a settled-and-re-registered successor is never + * evicted either. + */ + private List checkOwnershipAndPrune(Jedis jedis, String queueId, + Map leases, List> snapshot) { + final Set ids = new LinkedHashSet<>(snapshot.size()); + for (Map.Entry entry : snapshot) { + ids.add(entry.getKey()); + } + final Map owners = pendingOwners(jedis, queueId, ids); + final List mine = new ArrayList<>(snapshot.size()); + for (Map.Entry entry : snapshot) { + final StreamEntryID id = entry.getKey(); + final Lease lease = entry.getValue(); + if (lease.isReleaseDue()) { + // A delayed retry reached its deadline: release — stop renewing, let the + // idle clock run; the entry redelivers one visibility timeout later. + leases.remove(id, lease); + continue; + } + if (!lease.isHeldForRelease() && lease.ageNanos() > maxLeaseAgeNanos && !lease.isOwnerAlive()) { + leases.remove(id, lease); + metrics.leaseLeak(); + log.warn("Redis work queue - LEASE LEAK: queue={}; entry={}; age={} - a settlement path never ran and no live owner is bound; renewal stops so the claim cycle recovers the entry", queueId, id, Duration.ofNanos(lease.ageNanos())); + continue; + } + final String owner = owners.get(id); + if (owner == null) { + // Exact per-id answer: the entry is not pending at all — acked or deleted + // outside this lease. A settled lease racing its own un-registration is + // benign; an UNSETTLED one means someone else finished the entry (a + // consumer that took it over and completed, or an out-of-band ack) — + // count it as lost. + leases.remove(id, lease); + if (!lease.isSettled()) { + metrics.leaseLost(); + log.warn("Redis work queue - lease lost: queue={}; entry={} is no longer pending under an active lease - a duplicate execution is possible", queueId, id); + } + continue; + } + if (!owner.equals(consumerName)) { + leases.remove(id, lease); + metrics.leaseLost(); + log.warn("Redis work queue - lease lost: queue={}; entry={} is now owned by consumer={} - dropping it to avoid ownership ping-pong; a duplicate execution is possible", queueId, id, owner); + continue; + } + mine.add(id); + } + return mine; + } + + /** + * Map each leased entry to its current PEL owner: one pipelined per-id XPENDING — + * a single round-trip with exact answers, bounded by the leased count. + * + *

A range query capped by a count is NOT safe here: leased ids bracketing more + * foreign pending entries than the cap get truncated out of the response, and + * treating "missing" as "safe to renew" would re-seize an entry another consumer + * legitimately owns — a silent duplicate execution. With per-id queries an absent + * id has exactly one meaning: the entry is no longer pending. + */ + protected Map pendingOwners(Jedis jedis, String queueId, Set ids) { + final String group = config.getDefaultConsumerGroupName(); + final Pipeline pipeline = jedis.pipelined(); + final Map>> queries = new HashMap<>(); + for (StreamEntryID id : ids) { + queries.put(id, pipeline.xpending(queueId, group, new XPendingParams(id, id, 1))); } + pipeline.sync(); + final Map owners = new HashMap<>(); + for (Map.Entry>> query : queries.entrySet()) { + final List pending = query.getValue().get(); + if (pending != null && !pending.isEmpty()) { + owners.put(query.getKey(), pending.get(0).getConsumerName()); + } + } + return owners; } protected StreamEntry readMessage(Jedis jedis, String queueId) { @@ -278,7 +540,7 @@ protected StreamEntry readMessage(Jedis jedis, String queueId) { } } if (entry != null) { - log.trace("Redis queue id={}; read entry={}", queueId, entry); + log.trace("Redis work queue id={}; read entry={}", queueId, entry); } return entry; } @@ -335,7 +597,7 @@ protected StreamEntry claimMessage(Jedis jedis, String queueId) { ? messages.getValue().get(0) : null; if (entry != null) { - log.trace("Redis queue id={}; claimed entry={}", queueId, entry); + log.trace("Redis work queue id={}; claimed entry={}", queueId, entry); } return entry; } @@ -360,4 +622,136 @@ public int length(String queueId) { return (int) jedis.xlen(queueId); } } + + /** + * Settlement handle for one in-flight entry. Idempotent — the first {@code ack()} or + * {@code retry()} wins — and callable from any thread: {@code ack()} checks out its + * own pooled connection. + */ + private final class Lease implements MessageLease { + + private final String queueId; + + private final StreamEntryID entryId; + + /** Max-age backstop clock: the moment the entry was registered. */ + private final long registeredAt = System.nanoTime(); + + /** First settlement wins; every later call on either method is a no-op. */ + private final AtomicBoolean settled = new AtomicBoolean(); + + /** Owner-liveness probe bound by the lease taker; null when never bound. */ + private volatile BooleanSupplier liveness; + + /** + * Deadline (nanos) at which a delayed-retry lease is released to the normal + * redelivery clock; zero when the lease is not held for delayed retry. + */ + private volatile long releaseAtNanos; + + private Lease(String queueId, StreamEntryID entryId) { + this.queueId = queueId; + this.entryId = entryId; + } + + private long ageNanos() { + return System.nanoTime() - registeredAt; + } + + /** True only when a probe is bound and reports the owning task still running. */ + private boolean isOwnerAlive() { + final BooleanSupplier probe = liveness; + return probe != null && probe.getAsBoolean(); + } + + private boolean trySettle() { + return settled.compareAndSet(false, true); + } + + private boolean isSettled() { + return settled.get(); + } + + private void settleAsRetry() { + if (trySettle()) { + unregister(this); + } + } + + /** + * {@inheritDoc} + * + *

Un-register first, Redis second: a failed XACK then degrades to the benign + * case — the idle clock resumes, the entry redelivers, and the redelivery acks on + * the caller's terminal check — instead of a heartbeated-forever orphan. + */ + @Override + public void ack() { + if (!trySettle()) { + return; + } + unregister(this); + try (Jedis jedis = pool.getResource()) { + ackAndDelete(jedis, queueId, entryId); + } + catch (Exception e) { + log.warn("Redis work queue - ack failed for queue={}; entry={} - the entry will be redelivered and acked on the terminal check", queueId, entryId, e); + } + } + + /** + * {@inheritDoc} + * + *

Registry removal only: stopping renewal is the release, and the claim + * cadence is the retry schedule — no Redis call at all. + */ + @Override + public void retry() { + settleAsRetry(); + } + + /** + * {@inheritDoc} + * + *

The natural idle-out already takes one visibility timeout after release, so + * only the excess is held: the lease stays REGISTERED — renewed, never stalled — + * until {@code delay - visibility timeout} elapses, then the renewal tick releases + * it and the idle clock delivers at ≈ the requested delay. A held lease is + * settled (late {@code ack()}/{@code retry()} are no-ops) and is deliberate: + * the age backstop never prunes it as a leak. + */ + @Override + public void retryAfter(Duration delay) { + final long holdNanos = delay.toNanos() - TimeUnit.MILLISECONDS.toNanos(config.getVisibilityTimeoutMillis()); + if (holdNanos <= 0) { + retry(); + return; + } + if (settled.compareAndSet(false, true)) { + // A renewal tick racing this assignment sees a settled, registered, + // not-yet-held lease for at most one tick: it just renews it — benign. + releaseAtNanos = System.nanoTime() + holdNanos; + } + } + + private boolean isHeldForRelease() { + return releaseAtNanos > 0; + } + + private boolean isReleaseDue() { + return releaseAtNanos > 0 && System.nanoTime() >= releaseAtNanos; + } + + /** + * {@inheritDoc} + * + *

Gates the age backstop: while the probe reports the owning task alive, the + * lease is never pruned as a leak — a slow handler keeps its lease for as long + * as it actually runs. + */ + @Override + public void bindLiveness(BooleanSupplier alive) { + this.liveness = alive; + } + } } diff --git a/lib-data-workqueue-redis/src/main/java/io/seqera/data/workqueue/redis/RedisWorkQueueConfig.java b/lib-data-workqueue-redis/src/main/java/io/seqera/data/workqueue/redis/RedisWorkQueueConfig.java index 57207a2e..80819d10 100644 --- a/lib-data-workqueue-redis/src/main/java/io/seqera/data/workqueue/redis/RedisWorkQueueConfig.java +++ b/lib-data-workqueue-redis/src/main/java/io/seqera/data/workqueue/redis/RedisWorkQueueConfig.java @@ -20,8 +20,8 @@ import java.time.Duration; /** - * Configuration interface for Redis-backed work queues that defines timeout and consumer - * group settings for reliable message processing. + * Configuration interface for work queues that defines timeout and consumer group settings + * for queue-based message processing. * *

This interface provides configuration parameters for: *

    @@ -31,7 +31,7 @@ *
* *

Implementations should provide appropriate values based on the underlying - * work-queue technology (Redis Streams consumer groups) and application requirements. + * queue technology (e.g., Redis Streams) and application requirements. * *

Example usage: * 

{@code
@@ -44,12 +44,12 @@
  * }
* * @author Paolo Di Tommaso - * @since 1.0 + * @since 1.1 */ public interface RedisWorkQueueConfig { /** - * Returns the default consumer group name used when creating work queue consumers + * Returns the default consumer group name used when creating queue consumers * without an explicitly specified group. * * @return the default consumer group name, must not be null or empty @@ -57,11 +57,9 @@ public interface RedisWorkQueueConfig { String getDefaultConsumerGroupName(); /** - * Returns the visibility timeout duration for messages delivered from the queue. + * Returns the visibility timeout for messages handed to a consumer. * This timeout determines how long a consumer can hold a message before - * it becomes available for reclaiming by other consumers (mapped to the Redis - * consumer-group min-idle used by {@code XAUTOCLAIM}). Backed by the - * {@code visibility-timeout} configuration property. + * it becomes available for claiming by other consumers. * * @return the visibility timeout duration, must be positive */ @@ -97,46 +95,53 @@ default long getConsumerWarnTimeoutMillis() { } /** - * Returns how often in-flight leases are renewed (heartbeated) to keep them - * from being reclaimed by peer consumers while a handler is still running. - * Must be shorter than {@link #getVisibilityTimeout()}; defaults to {@code visibility-timeout / 3} - * so that up to two consecutive misses are tolerated. + * Period of the lease-renewal tick that keeps in-flight entries from going stalled. * - * @return the heartbeat interval duration + *

Defaults to a quarter of the visibility timeout, so the margin math tracks a + * re-tuned visibility timeout automatically: with a successful renewal at {@code t0}, + * ticks fire at {@code t0+P, t0+2P, t0+3P} while the entry becomes claimable at + * {@code t0+4P} — two consecutive failed or missed ticks are tolerated with a + * quarter-timeout margin remaining. Must be well below the visibility timeout: a + * period at or above it means every lease is claimable before its first renewal. + * + * @return the lease renewal period, must be positive and below the visibility timeout */ - default Duration getHeartbeatInterval() { - return getVisibilityTimeout().dividedBy(3); + default Duration getLeaseRenewalPeriod() { + return getVisibilityTimeout().dividedBy(4); } /** - * Returns the heartbeat interval in milliseconds for convenience. - * This is a derived value from {@link #getHeartbeatInterval()}. + * Returns the lease renewal period in milliseconds for convenience. + * This is a derived value from {@link #getLeaseRenewalPeriod()}. * - * @return the heartbeat interval in milliseconds + * @return the lease renewal period in milliseconds */ - default long getHeartbeatIntervalMillis() { - return getHeartbeatInterval().toMillis(); + default long getLeaseRenewalPeriodMillis() { + return getLeaseRenewalPeriod().toMillis(); } /** - * Returns the upper bound on a single {@code accept()} invocation before its - * lease is released (safety valve). This bounds one handler invocation, not the - * total lease lifetime; past this bound the heartbeat daemon stops renewing the - * lease so it becomes reclaimable. Defaults to {@code 15m}. + * Age past which an unsettled lease whose owner is not provably alive is treated as a + * registry leak: renewal stops, loudly, so the claim cycle can recover the entry. + * A lease whose bound owner is still running is never age-pruned, however long it + * runs — this bound only limits how long a leaked registration can keep an entry + * from going stalled. + * + *

Defaults to three visibility timeouts. * - * @return the maximum single-invocation processing time duration + * @return the maximum lease age, must be positive */ - default Duration getMaxProcessingTime() { - return Duration.ofMinutes(15); + default Duration getMaxLeaseAge() { + return getVisibilityTimeout().multipliedBy(3); } /** - * Returns the maximum processing time in milliseconds for convenience. - * This is a derived value from {@link #getMaxProcessingTime()}. + * Returns the maximum lease age in milliseconds for convenience. + * This is a derived value from {@link #getMaxLeaseAge()}. * - * @return the maximum processing time in milliseconds + * @return the maximum lease age in milliseconds */ - default long getMaxProcessingTimeMillis() { - return getMaxProcessingTime().toMillis(); + default long getMaxLeaseAgeMillis() { + return getMaxLeaseAge().toMillis(); } } diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/AsyncWorkQueueRedisTest.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/AsyncWorkQueueRedisTest.groovy deleted file mode 100644 index 7c28ab06..00000000 --- a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/AsyncWorkQueueRedisTest.groovy +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright 2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.data.workqueue - -import java.time.Duration -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -import io.micronaut.context.ApplicationContext -import io.seqera.data.workqueue.redis.RedisWorkQueue -import io.seqera.fixtures.redis.RedisTestContainer -import io.seqera.random.LongRndKey -import spock.lang.Specification - -/** - * Testcontainers-backed verification of the async lease model against a real Redis - * (consumer-group PEL semantics). Covers: no reclaim of live work (single- and - * two-instance), crash failover, and the max-processing-time safety valve. - * - * The test config sets {@code visibility-timeout = 1s}; the queues below heartbeat every - * 300ms so an alive owner keeps its lease. - * - * @author Paolo Di Tommaso - */ -class AsyncWorkQueueRedisTest extends Specification implements RedisTestContainer { - - private ApplicationContext newContext() { - // all contexts read the same redis.host/redis.port system properties set by the - // RedisTestContainer trait, so they share one Redis but each gets its own - // RedisWorkQueue bean (distinct consumer name) => independent instances - return ApplicationContext.run('test', 'redis') - } - - // single instance — a handler running longer than visibility-timeout is not - // reclaimed by this instance's own poll while it is alive/heartbeated - def 'should not reclaim live work within a single instance' () { - given: - def ctx = newContext() - def target = ctx.getBean(RedisWorkQueue) - // concurrency 2 keeps the dispatcher polling while the one message is in-flight - def queue = new TunableQueue(target, - concurrency: 2, - pollInterval: Duration.ofMillis(200), - heartbeatInterval: Duration.ofMillis(300)) - def id = "queue-${LongRndKey.rndHex()}" - def calls = new AtomicInteger() - def done = new CountDownLatch(1) - - when: - // handler runs ~3s (>> visibility-timeout 1s); the lease is heartbeated so it is never - // reclaimed and the dispatcher's own poll never re-delivers it - queue.addConsumer(id, { msg -> - calls.incrementAndGet() - Thread.sleep(3_000) - done.countDown() - true - }) - queue.offer(id, 'long-running') - - then: - done.await(8, TimeUnit.SECONDS) - and: - // give any spurious re-delivery a chance to show up, then assert single execution - sleep 1_000 - calls.get() == 1 - - cleanup: - queue.close() - ctx.stop() - } - - // two instances — a live, heartbeated owner is not reclaimed by a peer - def 'should not reclaim live work across two instances' () { - given: - def ctxA = newContext() - def ctxB = newContext() - def targetA = ctxA.getBean(RedisWorkQueue) - def targetB = ctxB.getBean(RedisWorkQueue) - def queueA = new TunableQueue(targetA, - concurrency: 1, pollInterval: Duration.ofMillis(200), heartbeatInterval: Duration.ofMillis(300)) - def queueB = new TunableQueue(targetB, - concurrency: 1, pollInterval: Duration.ofMillis(200), heartbeatInterval: Duration.ofMillis(300)) - def id = "queue-${LongRndKey.rndHex()}" - // shared across both instances: total number of times the message is processed - def calls = new AtomicInteger() - def done = new CountDownLatch(1) - - when: - def handler = { msg -> - calls.incrementAndGet() - Thread.sleep(3_000) // > visibility-timeout, but heartbeated -> no reclaim by peer - done.countDown() - true - } - queueA.addConsumer(id, handler) - queueB.addConsumer(id, handler) - queueA.offer(id, 'once') - - then: - done.await(8, TimeUnit.SECONDS) - and: - sleep 1_500 // longer than visibility-timeout, let any duplicate reclaim surface - calls.get() == 1 - - cleanup: - queueA.close() - queueB.close() - ctxA.stop() - ctxB.stop() - } - - // a non-heartbeating (crashed) owner's message is reclaimed by a peer after - // visibility-timeout and processed there - def 'should fail over to a peer when the owner stops heartbeating' () { - given: - def ctxDead = newContext() - def ctxLive = newContext() - def targetDead = ctxDead.getBean(RedisWorkQueue) - def targetLive = ctxLive.getBean(RedisWorkQueue) - // 'dead' owner: picks up the message and hangs, and never heartbeats (interval 1h) - // so its lease idle-time grows and becomes reclaimable after visibility-timeout - def queueDead = new TunableQueue(targetDead, - concurrency: 1, pollInterval: Duration.ofMillis(200), heartbeatInterval: Duration.ofHours(1)) - def queueLive = new TunableQueue(targetLive, - concurrency: 1, pollInterval: Duration.ofMillis(200), heartbeatInterval: Duration.ofMillis(300)) - def id = "queue-${LongRndKey.rndHex()}" - def hang = new CountDownLatch(1) - def deadStarted = new CountDownLatch(1) - def processedByLive = new CountDownLatch(1) - - when: 'only the dead owner is consuming, so it is guaranteed to pick up the message' - queueDead.addConsumer(id, { msg -> deadStarted.countDown(); hang.await(); true }) - queueDead.offer(id, 'orphan') - - then: 'the dead owner picks it up and then hangs without heartbeating' - deadStarted.await(5, TimeUnit.SECONDS) - - when: 'a live peer joins the group' - queueLive.addConsumer(id, { msg -> processedByLive.countDown(); true }) - - then: 'it reclaims the orphaned entry after the visibility timeout and processes it' - processedByLive.await(8, TimeUnit.SECONDS) - - cleanup: - hang.countDown() - queueDead.close() - queueLive.close() - ctxDead.stop() - ctxLive.stop() - } - - // a single invocation exceeding max-processing-time has its lease released - // (stops being renewed), so the message is reclaimed and re-delivered - def 'should release the lease of an invocation exceeding max-processing-time' () { - given: - def ctx = newContext() - def target = ctx.getBean(RedisWorkQueue) - def queue = new TunableQueue(target, - concurrency: 2, - pollInterval: Duration.ofMillis(200), - heartbeatInterval: Duration.ofMillis(300), - maxProcessingTime: Duration.ofSeconds(1)) - def id = "queue-${LongRndKey.rndHex()}" - def calls = new AtomicInteger() - def hang = new CountDownLatch(1) - def redelivered = new CountDownLatch(1) - - when: - queue.addConsumer(id, { msg -> - def n = calls.incrementAndGet() - if (n == 1) { - // first invocation hangs past max-processing-time (1s) -> lease released - hang.await() - return true - } - // the re-delivered invocation completes normally - redelivered.countDown() - return true - }) - queue.offer(id, 'hung') - - then: 'the stalled invocation is evicted and the message is re-delivered' - redelivered.await(10, TimeUnit.SECONDS) - calls.get() >= 2 - - cleanup: - hang.countDown() - queue.close() - ctx.stop() - } - -} diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TunableQueue.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TunableQueue.groovy deleted file mode 100644 index 4e2523a9..00000000 --- a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TunableQueue.groovy +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.data.workqueue - -import java.time.Duration - -import io.seqera.serde.encode.StringEncodingStrategy - -/** - * A {@link AbstractWorkQueue} used by the async-processing tests. It carries a - * String payload (identity encoding) and exposes the async knobs — concurrency, - * poll interval, heartbeat interval and max-processing-time — as constructor options - * so each test can tune them independently. - * - * @author Paolo Di Tommaso - */ -class TunableQueue extends AbstractWorkQueue { - - private final int workers - private final Duration pollDelay - private final Duration hbInterval - private final Duration maxProcTime - - TunableQueue(Map opts = [:], WorkQueue target) { - super(target) - withHandlerExecutor(TestWorkerPool.INSTANCE) - this.workers = (opts.concurrency ?: 1) as int - this.pollDelay = (opts.pollInterval ?: Duration.ofSeconds(1)) as Duration - this.hbInterval = (opts.heartbeatInterval ?: Duration.ofSeconds(20)) as Duration - this.maxProcTime = (opts.maxProcessingTime ?: Duration.ofMinutes(15)) as Duration - } - - @Override - protected StringEncodingStrategy createEncodingStrategy() { - return new StringEncodingStrategy() { - @Override - String encode(String message) { return message } - @Override - String decode(String encoded) { return encoded } - } - } - - @Override - protected String name() { - return 'tunable-queue' - } - - @Override - protected Duration pollInterval() { - return pollDelay - } - - @Override - protected int concurrency() { - return workers - } - - @Override - protected Duration heartbeatInterval() { - return hbInterval - } - - @Override - protected Duration maxProcessingTime() { - return maxProcTime - } -} diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueRedisTest.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/AbstractWorkQueueRedisTest.groovy similarity index 70% rename from lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueRedisTest.groovy rename to lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/AbstractWorkQueueRedisTest.groovy index 61ef6e9f..ce807164 100644 --- a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueRedisTest.groovy +++ b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/AbstractWorkQueueRedisTest.groovy @@ -15,7 +15,7 @@ * */ -package io.seqera.data.workqueue +package io.seqera.data.workqueue.redis import io.seqera.fixtures.redis.RedisTestContainer import io.seqera.random.LongRndKey @@ -25,7 +25,7 @@ import spock.lang.Specification import java.util.concurrent.ArrayBlockingQueue import io.micronaut.context.ApplicationContext -import io.seqera.data.workqueue.redis.RedisWorkQueue +import static io.seqera.data.workqueue.MessageConsumer.Decision.ACK /** * @@ -46,24 +46,24 @@ class AbstractWorkQueueRedisTest extends Specification implements RedisTestConta def 'should offer and consume some messages' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" + def id1 = "stream-${LongRndKey.rndHex()}" and: def target = context.getBean(RedisWorkQueue) - def queue = new TestQueue(target) - def sink = new ArrayBlockingQueue(10) + def stream = new TestQueue(target) + def queue = new ArrayBlockingQueue(10) and: - queue.addConsumer(id1, { it-> sink.add(it) }) + stream.addConsumer(id1, { it, lease -> queue.add(it); ACK }) when: - queue.offer(id1, new TestMessage('one','two')) - queue.offer(id1, new TestMessage('alpha','omega')) + stream.offer(id1, new TestMessage('one','two')) + stream.offer(id1, new TestMessage('alpha','omega')) then: - sink.take()==new TestMessage('one','two') - sink.take()==new TestMessage('alpha','omega') - + queue.take()==new TestMessage('one','two') + queue.take()==new TestMessage('alpha','omega') + cleanup: - queue.close() + stream.close() } } diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueConfigTest.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueConfigTest.groovy new file mode 100644 index 00000000..c851a2fa --- /dev/null +++ b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueConfigTest.groovy @@ -0,0 +1,121 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.workqueue.redis + +import java.time.Duration + +import spock.lang.Specification + +/** + * Covers the lease tuning settings of {@link RedisWorkQueueConfig}: the renewal period and + * the max lease age derive from the visibility timeout by default — so the margin math + * tracks a re-tuned visibility timeout — and can be overridden independently. + * + * @author Paolo Di Tommaso + */ +class RedisWorkQueueConfigTest extends Specification { + + static class DefaultsConfig implements RedisWorkQueueConfig { + @Override + String getDefaultConsumerGroupName() { 'test-group' } + @Override + Duration getVisibilityTimeout() { Duration.ofSeconds(60) } + @Override + Duration getConsumerWarnTimeout() { Duration.ofSeconds(40) } + } + + /** Defaults with explicit tuning overrides — null keeps the derived default. */ + static class TuningConfig extends DefaultsConfig { + Duration renewalPeriod + Duration leaseAge + @Override + Duration getLeaseRenewalPeriod() { renewalPeriod != null ? renewalPeriod : super.getLeaseRenewalPeriod() } + @Override + Duration getMaxLeaseAge() { leaseAge != null ? leaseAge : super.getMaxLeaseAge() } + } + + def 'lease settings should derive from the visibility timeout by default' () { + given: + def config = new DefaultsConfig() + + expect: 'renewal period is a quarter of the visibility timeout - two missed ticks tolerated with margin' + config.getLeaseRenewalPeriod() == Duration.ofSeconds(15) + and: 'the leak backstop is three visibility timeouts' + config.getMaxLeaseAge() == Duration.ofMinutes(3) + } + + def 'lease settings should be overridable independently of the visibility timeout' () { + given: + def config = new DefaultsConfig() { + @Override + Duration getLeaseRenewalPeriod() { Duration.ofSeconds(5) } + @Override + Duration getMaxLeaseAge() { Duration.ofMinutes(10) } + } + + expect: + config.getLeaseRenewalPeriodMillis() == 5_000 + config.getMaxLeaseAgeMillis() == 600_000 + } + + def 'a renewal period that cannot protect a lease should fail fast at startup' () { + given: 'a period >= the visibility timeout: every lease would be claimable before its first renewal' + def stream = new RedisWorkQueue() + stream.@config = new DefaultsConfig() { + @Override + Duration getLeaseRenewalPeriod() { Duration.ofSeconds(60) } + } + + when: + stream.create() + + then: + thrown(IllegalStateException) + } + + def 'a non-positive renewal period should fail fast instead of being clamped' () { + given: 'clamping to 1ms would mean ~1000 renewal pipelines per second against Redis' + def stream = new RedisWorkQueue() + stream.@config = new TuningConfig(renewalPeriod: period) + + when: + stream.create() + + then: + thrown(IllegalStateException) + + where: + period << [Duration.ZERO, Duration.ofSeconds(-5)] + } + + def 'a non-positive max lease age should fail fast instead of pruning everything' () { + given: 'a zero age would make every unbound lease a "leak" on its first tick' + def stream = new RedisWorkQueue() + stream.@config = new TuningConfig(leaseAge: age) + + when: + stream.create() + + then: + thrown(IllegalStateException) + + where: + age << [Duration.ZERO, Duration.ofSeconds(-1)] + } + +} diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueLeaseTest.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueLeaseTest.groovy new file mode 100644 index 00000000..5fc6734c --- /dev/null +++ b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueLeaseTest.groovy @@ -0,0 +1,528 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.workqueue.redis + +import java.time.Duration +import java.util.concurrent.atomic.AtomicInteger +import java.util.function.BooleanSupplier +import java.util.function.IntSupplier + +import io.micronaut.context.ApplicationContext +import io.seqera.data.workqueue.MessageLease +import io.seqera.data.workqueue.metrics.Outcome +import io.seqera.data.workqueue.metrics.QueueMetrics +import io.seqera.fixtures.redis.RedisTestContainer +import io.seqera.random.LongRndKey +import redis.clients.jedis.Jedis +import redis.clients.jedis.JedisPool +import redis.clients.jedis.StreamEntryID +import redis.clients.jedis.params.XClaimParams +import redis.clients.jedis.params.XPendingParams +import redis.clients.jedis.params.XReadGroupParams +import spock.lang.Shared +import spock.lang.Specification +import static io.seqera.data.workqueue.MessageConsumer.Decision.ACK +import static io.seqera.data.workqueue.MessageConsumer.Decision.DEFERRED + +/** + * Covers the message-lease (PEL heartbeat) semantics of {@link RedisWorkQueue}: + * a DEFERRED entry stays leased — invisible to other consumers — for as long as its + * lease is renewed, redelivers once the heartbeat dies, and renewal is batched so a + * single tick covers the whole in-flight set with one XPENDING + one XCLAIM. + * + * @author Paolo Di Tommaso + */ +class RedisWorkQueueLeaseTest extends Specification implements RedisTestContainer { + + static final Duration VISIBILITY_TIMEOUT = Duration.ofSeconds(2) + + static class LeaseTestConfig implements RedisWorkQueueConfig { + @Override + String getDefaultConsumerGroupName() { 'lease-test-group' } + @Override + Duration getVisibilityTimeout() { VISIBILITY_TIMEOUT } + @Override + Duration getConsumerWarnTimeout() { Duration.ofSeconds(10) } + } + + /** Counts the lease events the renewal tick reports. */ + static class RecordingMetrics implements QueueMetrics { + final AtomicInteger lost = new AtomicInteger() + final AtomicInteger leaks = new AtomicInteger() + final AtomicInteger errors = new AtomicInteger() + @Override + void bindBacklog(String queueId, IntSupplier lengthSupplier) { } + @Override + long startSample() { return 0 } + @Override + void recordOutcome(long startNanos, String queueId, Outcome outcome) { } + @Override + void leaseLost() { lost.incrementAndGet() } + @Override + void leaseLeak() { leaks.incrementAndGet() } + @Override + void renewError() { errors.incrementAndGet() } + } + + /** + * Interleaving hook: runs a callback right after the ownership query returns — + * the window in which the dispatcher can register a newborn lease concurrently. + */ + static class RacingQueue extends RedisWorkQueue { + Runnable onQuery + @Override + protected Map pendingOwners(Jedis jedis, String queueId, Set ids) { + final owners = super.pendingOwners(jedis, queueId, ids) + onQuery?.run() + return owners + } + } + + @Shared + ApplicationContext context + + List queues = [] + + def setup() { + context = ApplicationContext.run('test', 'redis') + } + + def cleanup() { + queues.each { (getInternal(it, 'renewalScheduler') as java.util.concurrent.ScheduledExecutorService)?.shutdownNow() } + queues.clear() + context.stop() + } + + private RedisWorkQueue newQueue(QueueMetrics metrics = null) { + return initQueue(new RedisWorkQueue(), metrics) + } + + /* Reflection accessors: Groovy's .@ direct field access cannot reach the private + superclass fields on a RacingQueue instance, so the harness goes through + java.lang.reflect for both the plain and the subclassed queue. */ + + private static void setInternal(RedisWorkQueue queue, String name, Object value) { + def field = RedisWorkQueue.getDeclaredField(name) + field.accessible = true + field.set(queue, value) + } + + private static Object getInternal(RedisWorkQueue queue, String name) { + def field = RedisWorkQueue.getDeclaredField(name) + field.accessible = true + return field.get(queue) + } + + private T initQueue(T queue, QueueMetrics metrics = null) { + setInternal(queue, 'pool', context.getBean(JedisPool)) + setInternal(queue, 'config', new LeaseTestConfig()) + setInternal(queue, 'metrics', metrics) + def create = RedisWorkQueue.getDeclaredMethod('create') + create.accessible = true + create.invoke(queue) + queues << queue + return queue + } + + private static boolean noLease(RedisWorkQueue queue, String queueId) { + final leases = queue.@inFlight.get(queueId) + return leases == null || leases.isEmpty() + } + + /** The current PEL owner of the given entry, or null when the entry is not pending. */ + private String pendingOwner(String queueId, StreamEntryID entryId) { + try (def jedis = context.getBean(JedisPool).getResource()) { + def pending = jedis.xpending(queueId, 'lease-test-group', new XPendingParams(entryId, entryId, 1)) + return pending ? pending.first().consumerName : null + } + } + + /** The idle time in millis of the given pending entry. */ + private long pendingIdle(String queueId, StreamEntryID entryId) { + try (def jedis = context.getBean(JedisPool).getResource()) { + def pending = jedis.xpending(queueId, 'lease-test-group', new XPendingParams(entryId, entryId, 1)) + return pending.first().idleTime + } + } + + def 'a deferred entry should stay leased past the visibility timeout and settle on ack' () { + given: 'two competing consumers with a 2s visibility timeout' + def stream1 = newQueue() + def stream2 = newQueue() + def queueId = "stream-${LongRndKey.rndHex()}" + stream1.init(queueId) + MessageLease held = null + + when: 'the first consumer defers the settlement to a task' + stream1.offer(queueId, 'payload') + def decision = stream1.consume(queueId, { msg, lease -> held = lease; DEFERRED }) + then: + decision == DEFERRED + held != null + + when: 'a second consumer keeps polling for 2.5x the visibility timeout' + def stalled = false + def deadline = System.currentTimeMillis() + VISIBILITY_TIMEOUT.toMillis() * 5 / 2 + while (System.currentTimeMillis() < deadline) { + if (stream2.consume(queueId, { msg, lease -> ACK }) != null) { + stalled = true + break + } + sleep 250 + } + then: 'the heartbeat kept the entry invisible - no second delivery' + !stalled + + when: 'the task settles from another thread' + def settler = new Thread({ held.ack() }) + settler.start() + settler.join() + then: 'the entry is acked, removed and unregistered' + stream1.length(queueId) == 0 + noLease(stream1, queueId) + stream2.consume(queueId, { msg, lease -> assert false /* <-- this should not be invoked */ }) == null + } + + def 'a dead lease should be redelivered to another consumer after the visibility timeout' () { + given: + def stream1 = newQueue() + def stream2 = newQueue() + def queueId = "stream-${LongRndKey.rndHex()}" + stream1.init(queueId) + + when: 'the first consumer defers and then its replica "crashes" - renewal stops, the lease is never settled' + stream1.offer(queueId, 'payload') + stream1.consume(queueId, { msg, lease -> DEFERRED }) + stream1.@renewalScheduler.shutdownNow() + + and: 'a second consumer polls past the visibility timeout' + String redelivered = null + def deadline = System.currentTimeMillis() + VISIBILITY_TIMEOUT.toMillis() * 5 + while (redelivered == null && System.currentTimeMillis() < deadline) { + stream2.consume(queueId, { msg, lease -> redelivered = msg; ACK }) + sleep 250 + } + + then: 'the entry idles out and is claimed exactly like a dead-consumer failure' + redelivered == 'payload' + stream1.length(queueId) == 0 + } + + def 'one renewal tick should cover many in-flight entries in a single batch' () { + given: 'a stream with the background renewal stopped, so only manual ticks renew' + def stream = newQueue() + stream.@renewalScheduler.shutdownNow() + def queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + def count = 40 + + and: 'many entries in flight, all deferred' + count.times { stream.offer(queueId, "msg-$it".toString()) } + count.times { + assert stream.consume(queueId, { msg, lease -> DEFERRED }) == DEFERRED + } + + when: 'idle accumulates, then one manual renewal tick runs' + sleep 1_000 + def begin = System.nanoTime() + stream.renewLeases() + def elapsed = Duration.ofNanos(System.nanoTime() - begin) + + and: 'the pending entries are inspected right after the tick' + def pending + try (def jedis = context.getBean(JedisPool).getResource()) { + pending = jedis.xpending(queueId, 'lease-test-group', new XPendingParams('-', '+', count * 2)) + } + + then: 'the single tick renewed every entry - idle time was reset for all of them' + pending.size() == count + pending.every { it.idleTime < 800 } + + and: 'the tick is two round-trips, not one per entry - far below the renewal period' + elapsed < Duration.ofMillis(VISIBILITY_TIMEOUT.toMillis().intdiv(4)) + } + + def 'retry should release the lease and let the entry redeliver on the claim cadence' () { + given: + def stream = newQueue() + def queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + MessageLease held = null + + when: + stream.offer(queueId, 'payload') + stream.consume(queueId, { msg, lease -> held = lease; DEFERRED }) + and: 'the task settles as retry - registry removal only, the entry stays pending' + held.retry() + then: + noLease(stream, queueId) + stream.length(queueId) == 1 + + when: 'a late double-settlement is a no-op' + held.ack() + then: + stream.length(queueId) == 1 + + when: 'the claim cycle re-delivers after the visibility timeout' + String redelivered = null + def deadline = System.currentTimeMillis() + VISIBILITY_TIMEOUT.toMillis() * 5 + while (redelivered == null && System.currentTimeMillis() < deadline) { + stream.consume(queueId, { msg, lease -> redelivered = msg; ACK }) + sleep 250 + } + then: + redelivered == 'payload' + stream.length(queueId) == 0 + } + + def 'retryAfter should hold the lease and redeliver no earlier than the delay' () { + given: 'two consumers; the first defers and settles with a delayed retry' + def stream1 = newQueue() + def stream2 = newQueue() + def queueId = "stream-${LongRndKey.rndHex()}" + stream1.init(queueId) + MessageLease held = null + def delay = VISIBILITY_TIMEOUT.multipliedBy(5).dividedBy(2) // 5s for a 2s visibility timeout + + when: + stream1.offer(queueId, 'payload') + stream1.consume(queueId, { msg, lease -> held = lease; DEFERRED }) + held.retryAfter(delay) + + and: 'a second consumer polls the whole window' + long deliveredAt = 0 + def begin = System.currentTimeMillis() + def deadline = begin + delay.toMillis() * 3 + while (deliveredAt == 0 && System.currentTimeMillis() < deadline) { + if (stream2.consume(queueId, { msg, lease -> ACK }) != null) { + deliveredAt = System.currentTimeMillis() + } + sleep 200 + } + + then: 'the entry redelivers, but no earlier than the requested delay' + deliveredAt > 0 + deliveredAt - begin >= delay.toMillis() - 500 // scheduling slop + } + + def 'a lease held for delayed retry should never be pruned as a leak' () { + given: 'a consumer whose renewal runs only on manual ticks, with recording metrics' + def metrics = new RecordingMetrics() + def stream = newQueue(metrics) + stream.@renewalScheduler.shutdownNow() + String queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + MessageLease held = null + + when: 'a deferred entry settles with a delay far beyond the age backstop' + stream.offer(queueId, 'payload') + stream.consume(queueId, { msg, lease -> held = lease; DEFERRED }) + held.retryAfter(VISIBILITY_TIMEOUT.multipliedBy(30)) + and: 'the lease ages past 3x the visibility timeout, then a renewal tick runs' + sleep VISIBILITY_TIMEOUT.toMillis() * 3 + 500 + stream.renewLeases() + + then: 'held for release on purpose - not a leak, not lost, still registered and renewed' + metrics.leaks.get() == 0 + metrics.lost.get() == 0 + stream.@inFlight.get(queueId).size() == 1 + } + + def 'a renewal tick against an exhausted pool should fail fast and loudly, not hang' () { + given: 'a leased entry, then a bounded-borrow pool fully exhausted by other borrowers' + def metrics = new RecordingMetrics() + def stream = newQueue(metrics) + (getInternal(stream, 'renewalScheduler') as java.util.concurrent.ScheduledExecutorService).shutdownNow() + String queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + stream.offer(queueId, 'payload') + stream.consume(queueId, { msg, lease -> DEFERRED }) + and: 'every pool connection borrowed away, with a bounded borrow wait (redis.pool.maxWait in prod)' + def pool = context.getBean(JedisPool) + pool.setMaxWait(java.time.Duration.ofMillis(500)) + def borrowed = [] + pool.maxTotal.times { borrowed << pool.getResource() } + + when: 'a renewal tick runs while no connection can be borrowed' + def begin = System.currentTimeMillis() + stream.renewLeases() + def elapsed = System.currentTimeMillis() - begin + + then: 'the tick completed within the bound and reported the failure - no silent hang' + elapsed < 5_000 + metrics.errors.get() >= 1 + and: 'the lease stays registered - the next tick retries the renewal' + (getInternal(stream, 'inFlight') as Map).get(queueId).size() == 1 + + cleanup: + borrowed.each { it.close() } + pool.setMaxWait(java.time.Duration.ofMillis(-1)) + } + + def 'the renewal liveness gauge should expose the age of the last completed tick' () { + given: 'a stream instrumented with real Micrometer metrics, ticking only manually' + def registry = new io.micrometer.core.instrument.simple.SimpleMeterRegistry() + def stream = initQueue(new RedisWorkQueue(), + new io.seqera.data.workqueue.metrics.MicrometerQueueMetrics(registry, 'lease-liveness-test')) + (getInternal(stream, 'renewalScheduler') as java.util.concurrent.ScheduledExecutorService).shutdownNow() + + expect: 'the gauge is registered at stream creation' + def gauge = registry.find('seqera.workqueue.lease.renewal.age').gauge() + gauge != null + + when: 'time passes, then a manual tick completes' + sleep 400 + def before = gauge.value() + stream.renewLeases() + + then: 'the age was growing and the completed tick reset it - a STUCK tick shows unbounded growth' + before >= 0.35d + gauge.value() < before + } + + def 'a throwing consumer should leave nothing registered' () { + given: + def stream = newQueue() + def queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + + when: + stream.offer(queueId, 'payload') + stream.consume(queueId, { msg, lease -> throw new RuntimeException('Oops') }) + then: + def e = thrown(RuntimeException) + e.message == 'Oops' + and: 'the registration bracket settled the entry as retry' + noLease(stream, queueId) + stream.length(queueId) == 1 + } + + def 'a lease taken over by another consumer should be dropped on the next tick and never re-seized' () { + given: 'a consumer whose renewal runs only on manual ticks, with recording metrics' + def metrics = new RecordingMetrics() + def stream = newQueue(metrics) + stream.@renewalScheduler.shutdownNow() + String queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + + when: 'an entry is deferred, then force-claimed by another consumer - as after a renewal outage' + stream.offer(queueId, 'payload') + stream.consume(queueId, { msg, lease -> DEFERRED }) + StreamEntryID entryId = stream.@inFlight.get(queueId).keySet().first() + try (def jedis = context.getBean(JedisPool).getResource()) { + jedis.xclaim(queueId, 'lease-test-group', 'thief-consumer', 0, new XClaimParams(), entryId) + } + and: 'the next renewal tick runs' + stream.renewLeases() + + then: 'the ownership check dropped the lease and counted it as lost' + noLease(stream, queueId) + metrics.lost.get() == 1 + and: 'the thief still owns the entry - no re-seizure ping-pong' + pendingOwner(queueId, entryId) == 'thief-consumer' + } + + def 'a lease registered during the ownership check must not be dropped' () { + given: 'a stream whose ownership query is raced by a concurrent registration' + def metrics = new RecordingMetrics() + def stream = initQueue(new RacingQueue(), metrics) + (getInternal(stream, 'renewalScheduler') as java.util.concurrent.ScheduledExecutorService).shutdownNow() + String queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + + and: 'one leased entry, and a second message not yet delivered' + stream.offer(queueId, 'first') + assert stream.consume(queueId, { msg, lease -> DEFERRED }) == DEFERRED + stream.offer(queueId, 'second') + + and: 'the dispatcher races the tick: it claims and registers the second entry right after the ownership query returned' + stream.onQuery = { stream.consume(queueId, { msg, lease -> DEFERRED }) } + + when: + stream.renewLeases() + + then: 'the newborn lease survives the tick untouched - it renews on the NEXT tick' + metrics.lost.get() == 0 + (getInternal(stream, 'inFlight') as Map).get(queueId).size() == 2 + } + + def 'the ownership check should stay exact under a large foreign PEL' () { + given: 'a consumer whose renewal runs only on manual ticks, with recording metrics' + def metrics = new RecordingMetrics() + def stream = newQueue(metrics) + stream.@renewalScheduler.shutdownNow() + String queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + + and: 'a leased entry, then 300 entries pending for ANOTHER consumer of the same group, then a second leased entry' + stream.offer(queueId, 'low') + assert stream.consume(queueId, { msg, lease -> DEFERRED }) == DEFERRED + 300.times { stream.offer(queueId, "foreign-$it".toString()) } + try (def jedis = context.getBean(JedisPool).getResource()) { + // deliver the 300 to a different consumer: they become foreign PEL entries + // interleaved between the ids this stream holds leases on + jedis.xreadGroup('lease-test-group', 'other-consumer', + new XReadGroupParams().count(300), Map.of(queueId, StreamEntryID.UNRECEIVED_ENTRY)) + } + stream.offer(queueId, 'high') + assert stream.consume(queueId, { msg, lease -> DEFERRED }) == DEFERRED + + and: 'the high leased entry goes stalled and is taken over, as after a renewal outage' + StreamEntryID highId = (stream.@inFlight.get(queueId).keySet() as List).max() + try (def jedis = context.getBean(JedisPool).getResource()) { + jedis.xclaim(queueId, 'lease-test-group', 'thief-consumer', 0, new XClaimParams(), highId) + } + + when: 'a renewal tick runs with the foreign entries crowding the leased id range' + stream.renewLeases() + + then: 'the theft was detected despite the crowded range - dropped and counted, never re-seized' + metrics.lost.get() == 1 + pendingOwner(queueId, highId) == 'thief-consumer' + stream.@inFlight.get(queueId).size() == 1 + } + + def 'the age backstop should prune a dead-owner lease as a leak but never a live one' () { + given: 'a consumer whose renewal runs only on manual ticks, with recording metrics' + def metrics = new RecordingMetrics() + def stream = newQueue(metrics) + stream.@renewalScheduler.shutdownNow() + String queueId = "stream-${LongRndKey.rndHex()}" + stream.init(queueId) + def leases = [:] + + and: 'two deferred entries: one bound to a live owner, one whose settlement path never ran' + stream.offer(queueId, 'alive') + stream.offer(queueId, 'leaked') + 2.times { + assert stream.consume(queueId, { msg, lease -> leases[msg] = lease; DEFERRED }) == DEFERRED + } + leases['alive'].bindLiveness({ true } as BooleanSupplier) + + when: 'both leases age past 3x the visibility timeout, then a renewal tick runs' + sleep VISIBILITY_TIMEOUT.toMillis() * 3 + 500 + stream.renewLeases() + + then: 'the dead-owner lease was pruned as a leak; the live one kept its lease' + metrics.leaks.get() == 1 + stream.@inFlight.get(queueId).values() as List == [leases['alive']] + and: 'the live lease was actually renewed - its idle clock was just reset' + pendingIdle(queueId, leases['alive'].@entryId as StreamEntryID) < 800 + } + +} diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/RedisWorkQueueTest.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueTest.groovy similarity index 51% rename from lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/RedisWorkQueueTest.groovy rename to lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueTest.groovy index b8507b07..24c498b7 100644 --- a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/RedisWorkQueueTest.groovy +++ b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/RedisWorkQueueTest.groovy @@ -15,15 +15,16 @@ * */ -package io.seqera.data.workqueue +package io.seqera.data.workqueue.redis import io.seqera.random.LongRndKey import spock.lang.Shared import spock.lang.Specification import io.micronaut.context.ApplicationContext -import io.seqera.data.workqueue.redis.RedisWorkQueue import io.seqera.fixtures.redis.RedisTestContainer +import static io.seqera.data.workqueue.MessageConsumer.Decision.ACK +import static io.seqera.data.workqueue.MessageConsumer.Decision.RETRY /** * @@ -44,112 +45,112 @@ class RedisWorkQueueTest extends Specification implements RedisTestContainer { def 'should offer and consume a value' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" - def id2 = "queue-${LongRndKey.rndHex()}" + def id1 = "stream-${LongRndKey.rndHex()}" + def id2 = "stream-${LongRndKey.rndHex()}" and: - def queue = context.getBean(RedisWorkQueue) + def stream = context.getBean(RedisWorkQueue) and: - queue.init(id1) - queue.init(id2) + stream.init(id1) + stream.init(id2) when: - queue.offer(id1, 'one') + stream.offer(id1, 'one') and: - queue.offer(id2, 'alpha') - queue.offer(id2, 'delta') - queue.offer(id2, 'gamma') + stream.offer(id2, 'alpha') + stream.offer(id2, 'delta') + stream.offer(id2, 'gamma') then: - queue.consume(id1, { it-> it=='one'}) + stream.consume(id1, { it, lease -> assert it=='one'; ACK }) == ACK and: - queue.consume(id2, { it-> it=='alpha'}) - queue.consume(id2, { it-> it=='delta'}) - queue.consume(id2, { it-> it=='gamma'}) + stream.consume(id2, { it, lease -> assert it=='alpha'; ACK }) == ACK + stream.consume(id2, { it, lease -> assert it=='delta'; ACK }) == ACK + stream.consume(id2, { it, lease -> assert it=='gamma'; ACK }) == ACK and: - !queue.consume(id2, { it-> assert false /* <-- this should not be invoked */ }) + stream.consume(id2, { it, lease -> assert false /* <-- this should not be invoked */ }) == null } def 'should offer and consume a value with a failure' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" - def queue = context.getBean(RedisWorkQueue) - queue.init(id1) + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = context.getBean(RedisWorkQueue) + stream.init(id1) when: - queue.offer(id1, 'alpha') - queue.offer(id1, 'delta') - queue.offer(id1, 'gamma') + stream.offer(id1, 'alpha') + stream.offer(id1, 'delta') + stream.offer(id1, 'gamma') then: - queue.consume(id1, { it-> it=='alpha'}) + stream.consume(id1, { it, lease -> assert it=='alpha'; ACK }) == ACK and: try { - queue.consume(id1, { it-> throw new RuntimeException("Oops")}) + stream.consume(id1, { it, lease -> throw new RuntimeException("Oops") }) } catch (RuntimeException e) { assert e.message == 'Oops' } and: // next message is 'gamma' as expected - queue.consume(id1, { it-> it=='gamma'}) + stream.consume(id1, { it, lease -> assert it=='gamma'; ACK }) == ACK and: // still nothing - !queue.consume(id1, { it-> assert false /* <-- this should not be invoked */ }) + stream.consume(id1, { it, lease -> assert false /* <-- this should not be invoked */ }) == null and: - // wait 2 seconds (visibility timeout is 1 sec) + // wait 2 seconds (claim timeout is 1 sec) sleep 2_000 // now the errored message is available - queue.consume(id1, { it-> it=='delta'}) + stream.consume(id1, { it, lease -> assert it=='delta'; ACK }) == ACK and: - !queue.consume(id1, { it-> assert false /* <-- this should not be invoked */ }) + stream.consume(id1, { it, lease -> assert false /* <-- this should not be invoked */ }) == null when: - queue.offer(id1, 'something') + stream.offer(id1, 'something') then: - queue.consume(id1, { it-> it=='something'}) + stream.consume(id1, { it, lease -> assert it=='something'; ACK }) == ACK } def 'should validate length method' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" - def queue = context.getBean(RedisWorkQueue) - queue.init(id1) + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = context.getBean(RedisWorkQueue) + stream.init(id1) expect: - queue.length(id1) == 0 + stream.length(id1) == 0 when: - queue.offer(id1, 'alpha') - queue.offer(id1, 'delta') - queue.offer(id1, 'gamma') + stream.offer(id1, 'alpha') + stream.offer(id1, 'delta') + stream.offer(id1, 'gamma') then: - queue.length(id1) == 3 + stream.length(id1) == 3 when: - queue.consume(id1, { it-> true}) + stream.consume(id1, { it, lease -> ACK }) then: - queue.length(id1) == 2 + stream.length(id1) == 2 } def 'should claim messages in round-robin fashion to prevent starvation' () { - given: 'a queue with multiple messages' - def queueId = "queue-${LongRndKey.rndHex()}" - def queue = context.getBean(RedisWorkQueue) - queue.init(queueId) + given: 'a stream with multiple messages' + def queueId = "stream-${LongRndKey.rndHex()}" + def stream = context.getBean(RedisWorkQueue) + stream.init(queueId) and: 'track which messages are consumed' def consumedMessages = Collections.synchronizedList([]) - when: 'add 5 messages to the queue' - queue.offer(queueId, 'msg-1') - queue.offer(queueId, 'msg-2') - queue.offer(queueId, 'msg-3') - queue.offer(queueId, 'msg-4') - queue.offer(queueId, 'msg-5') + when: 'add 5 messages to the stream' + stream.offer(queueId, 'msg-1') + stream.offer(queueId, 'msg-2') + stream.offer(queueId, 'msg-3') + stream.offer(queueId, 'msg-4') + stream.offer(queueId, 'msg-5') - and: 'consume all messages but reject them (return false) - simulating RUNNING tasks' - // First pass - read all messages, reject all (they go to PEL) + and: 'consume all messages but leave them pending (RETRY) - simulating RUNNING tasks' + // First pass - read all messages, retry all (they go to PEL) 5.times { - queue.consume(queueId, { msg -> + stream.consume(queueId, { msg, lease -> consumedMessages << msg - return false // reject - message stays in PEL + return RETRY // leave pending - message stays in PEL }) } @@ -157,16 +158,16 @@ class RedisWorkQueueTest extends Specification implements RedisTestContainer { consumedMessages.size() == 5 consumedMessages.containsAll(['msg-1', 'msg-2', 'msg-3', 'msg-4', 'msg-5']) - when: 'clear tracking and wait for visibility timeout' + when: 'clear tracking and wait for claim timeout' consumedMessages.clear() - sleep 1500 // visibility timeout is 1 second in test config + sleep 1500 // claim timeout is 1 second in test config and: 'consume again multiple times - messages should be reclaimed in round-robin' // Consume 10 times to verify round-robin (should see each message ~2 times) 10.times { - queue.consume(queueId, { msg -> + stream.consume(queueId, { msg, lease -> consumedMessages << msg - return false // keep rejecting + return RETRY // keep leaving them pending }) sleep 100 // small delay between consumes } diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestConfig.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestConfig.groovy similarity index 89% rename from lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestConfig.groovy rename to lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestConfig.groovy index 0926b4a5..a0493b8b 100644 --- a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestConfig.groovy +++ b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestConfig.groovy @@ -15,12 +15,11 @@ * */ -package io.seqera.data.workqueue +package io.seqera.data.workqueue.redis import java.time.Duration import io.micronaut.context.annotation.Requires -import io.seqera.data.workqueue.redis.RedisWorkQueueConfig import jakarta.inject.Singleton /** * @@ -32,7 +31,7 @@ class TestConfig implements RedisWorkQueueConfig { @Override String getDefaultConsumerGroupName() { - return "wave-work-queue" + return "wave-message-stream" } @Override diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestMessage.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestMessage.groovy similarity index 95% rename from lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestMessage.groovy rename to lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestMessage.groovy index 51485649..9fbaf547 100644 --- a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestMessage.groovy +++ b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestMessage.groovy @@ -15,7 +15,7 @@ * */ -package io.seqera.data.workqueue +package io.seqera.data.workqueue.redis import groovy.transform.Canonical diff --git a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestQueue.groovy b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestQueue.groovy similarity index 93% rename from lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestQueue.groovy rename to lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestQueue.groovy index fa5f61fa..59089be7 100644 --- a/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/TestQueue.groovy +++ b/lib-data-workqueue-redis/src/test/groovy/io/seqera/data/workqueue/redis/TestQueue.groovy @@ -15,11 +15,13 @@ * */ -package io.seqera.data.workqueue +package io.seqera.data.workqueue.redis import java.time.Duration import io.micrometer.core.instrument.MeterRegistry +import io.seqera.data.workqueue.AbstractWorkQueue +import io.seqera.data.workqueue.WorkQueue import io.seqera.data.workqueue.metrics.MicrometerQueueMetrics import io.seqera.data.workqueue.metrics.QueueMetrics import io.seqera.serde.encode.StringEncodingStrategy @@ -34,12 +36,10 @@ class TestQueue extends AbstractWorkQueue { TestQueue(WorkQueue target) { super(target) - withHandlerExecutor(TestWorkerPool.INSTANCE) } TestQueue(WorkQueue target, QueueMetrics metrics) { super(target, metrics) - withHandlerExecutor(TestWorkerPool.INSTANCE) } static TestQueue withRegistry(WorkQueue target, MeterRegistry registry) { @@ -53,7 +53,7 @@ class TestQueue extends AbstractWorkQueue { String encode(TestMessage message) { return new JsonBuilder([x: message.x, y: message.y]).toString() } - + @Override TestMessage decode(String encoded) { def json = new JsonSlurper().parseText(encoded) diff --git a/lib-data-workqueue-redis/src/test/java/io/seqera/data/workqueue/TestWorkerPool.java b/lib-data-workqueue-redis/src/test/java/io/seqera/data/workqueue/TestWorkerPool.java deleted file mode 100644 index af5bd1ef..00000000 --- a/lib-data-workqueue-redis/src/test/java/io/seqera/data/workqueue/TestWorkerPool.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.data.workqueue; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * Shared daemon handler executor for tests. {@link AbstractWorkQueue} no longer ships a - * built-in default executor (handlers must be supplied via {@code withHandlerExecutor}), so the - * test fixtures inject this one. Daemon threads so it never keeps the test JVM alive. - */ -public final class TestWorkerPool { - private TestWorkerPool() {} - - public static final ExecutorService INSTANCE = Executors.newCachedThreadPool(r -> { - Thread t = new Thread(r, "test-handler"); - t.setDaemon(true); - return t; - }); -} diff --git a/lib-data-workqueue/README.md b/lib-data-workqueue/README.md index 8a1abc7f..e19e7e9b 100644 --- a/lib-data-workqueue/README.md +++ b/lib-data-workqueue/README.md @@ -1,28 +1,85 @@ # lib-data-workqueue -A distributed, reliable **work queue** abstraction with competing consumers, acknowledgment -and lease/visibility-timeout semantics — plus an in-memory implementation for local and test -use. The Redis implementation lives in the companion module -[`lib-data-workqueue-redis`](../lib-data-workqueue-redis). - -> **Migrating from `lib-data-stream-redis`?** This module (together with -> `lib-data-workqueue-redis`) is the split/rename of `lib-data-stream-redis` 1.6.0. It keeps -> the exact behaviour and only renames the abstraction to match its real semantics. See the -> full guide at -> [`docs/superpowers/specs/2026-07-11-workqueue-rename-migration.md`](../docs/superpowers/specs/2026-07-11-workqueue-rename-migration.md). +A distributed, reliable **work queue**: competing consumers, one live owner per entry, +acknowledgment, a lease with heartbeat renewal, redelivery and dead-owner reclaim — plus an +in-memory implementation for local and test use. The Redis backend lives in the companion +module [`lib-data-workqueue-redis`](../lib-data-workqueue-redis/README.md). + +> **Supersedes `lib-data-stream-redis`.** This module is the reliable work queue that +> [#86](https://github.com/seqeralabs/libseqera/pull/86) renamed into place and +> [#100](https://github.com/seqeralabs/libseqera/pull/100) then left unpublished as a +> placeholder, "because [it carries] the lease-based design forward". 2.0.0 fills that +> placeholder with the implementation the scheduler has been running: a +> `MessageConsumer.Decision` returned from `consume()`, a `MessageLease` settlement handle, +> a cooperative drain, and heartbeat renewal pushed down into the Redis backend. +> +> **2.0.0 is a breaking SPI change over the unpublished 1.0.0**, which exposed +> `receive`/`renewLease`/`ack`/`release` around a `Lease` *record* and drove handlers from +> an executor and semaphore owned by `AbstractWorkQueue`. This version exposes +> `init`/`offer`/`consume`/`length`, returns a `Decision` from `consume()`, and hands the +> consumer a `MessageLease` *interface*. There is no migration path between the two; nothing +> in production consumed 1.0.0. +> +> The published `io.seqera:lib-data-stream-redis` artifact still exists and is still used by +> other services on older pinned versions; those are unaffected by anything here. ## Installation -Add this dependency to your `build.gradle`: - ```gradle dependencies { - implementation 'io.seqera:lib-data-workqueue:1.0.0' + implementation 'io.seqera:lib-data-workqueue:2.0.0' // for the Redis-backed implementation also add: - // implementation 'io.seqera:lib-data-workqueue-redis:1.0.0' + // implementation 'io.seqera:lib-data-workqueue-redis:2.0.0' } ``` +The library is pure Java: no Groovy runtime dependency, and no jedis — Redis lives entirely in +`lib-data-workqueue-redis`. + +## Cooperative shutdown + +`AbstractWorkQueue` never interrupts its dispatcher thread: an interrupt landing in a Redis +read can hand a RESP-desynced connection back to the pool (libseqera#92), and it was observed +propagating into a consumer still doing useful work. Shutdown is flag-based instead: + +- **`awaitQuiescent(timeout)`** — stop claiming new messages and wait for the dispatcher to finish + the message it holds, *without* releasing the queue. Call this first when consumers need + collaborators (a datasource, for instance) that are about to be torn down; returns `false` if the + dispatcher is still running at the deadline, and the caller decides what that means. +- **`close(timeout)`** — cooperative stop bounded by the *caller's* remaining budget, for callers + that already spent part of an overall shutdown budget on a drain. Never interrupts: the `closing` + flag guarantees the dispatcher exits at its next loop-head check, and the thread is a daemon. +- **`close()`** — same, with the `closeTimeout()` default (10s). **Only the first close waits**; + repeated calls — an explicit drain followed by a `@PreDestroy` backstop — return immediately, so + a shutdown budget is never spent twice. + +## Message leases + +`MessageConsumer.accept(message, lease)` returns a `Decision` instead of a boolean: + +- **`ACK`** — settle now: the entry is acknowledged and removed from the queue. +- **`RETRY`** — leave pending: the entry redelivers after the visibility timeout. A thrown + exception settles the same way (and propagates to the caller). +- **`DEFERRED`** — a task now owns the entry via the `MessageLease` handle and settles it + later with `lease.ack()` or `lease.retry()`, from any thread. Settlement is idempotent — + first call wins. + +On the Redis implementation a delivered entry is *leased*: a single background scheduler +renews every in-flight entry per queue in one round-trip (`XPENDING` ownership check + +one variadic `XCLAIM JUSTID`) at `visibility-timeout / 4`, so a live consumer can run past +the visibility timeout without the entry being stolen — the visibility timeout detects dead +consumers only. Leases stolen during a renewal outage are dropped (never re-seized) and +counted; leases older than 3× the visibility timeout whose owner is not provably alive are +treated as registry leaks and released to the claim cycle — `lease.bindLiveness(probe)` binds +the owning task's liveness (typically `() -> !task.isDone()`), and a lease whose probe reports +the owner alive is never age-pruned, however long it runs. The local implementation +mirrors the settlement semantics in-memory, with no visibility clock: a `RETRY` is +redelivered after a short delay (`workqueue.local.retry-delay`, default 1s). + +`MessageConsumer.ready()` (default `true`) is an admission gate: the dispatcher does not +claim from a queue while its consumer reports not ready; skipped polls count as +`saturated` in the metrics. + ## Metrics (optional) `AbstractWorkQueue` can publish [Micrometer](https://micrometer.io/) metrics when a @@ -48,6 +105,12 @@ references `MeterRegistry`, so subclasses that don't want metrics (using the 1-a constructor) can be loaded and instantiated even when `micrometer-core` is absent from the classpath. +The lease meters (`seqera.workqueue.leased`, `seqera.workqueue.lease.*`) are recorded one layer +below, by `RedisWorkQueue`, which injects an *optional* `QueueMetrics` bean from the +DI context and falls back to a no-op when none exists. Deployments that want the lease +metrics must therefore provide a `QueueMetrics` bean, typically from the same factory that +builds the queue and guarded on a `MeterRegistry` being present. + When enabled, the following meters are published. All meters carry the base tags `queue` (the subclass `name()`, e.g. `cmd-queue`) and `queue_id` (the actual Redis stream key, e.g. `cmd-queue/v1`). @@ -57,14 +120,23 @@ stream key, e.g. `cmd-queue/v1`). | `seqera.workqueue.entries` | Gauge | — | entries | Current queue backlog (Redis `XLEN`, polled at scrape time). | | `seqera.workqueue.messages` | Counter | `outcome` | messages | Total messages processed per outcome. | | `seqera.workqueue.processing` | Timer | `outcome` | seconds | Per-entry processing time. Includes the full lifecycle from the underlying `queue.consume(...)` entry through the consumer's `accept` and the Redis acknowledge/delete. Published as a Prometheus histogram (with buckets) so quantiles can be aggregated server-side across replicas via `histogram_quantile()`. | +| `seqera.workqueue.deferred` | Counter | — | — | Deliveries whose consumer returned `DEFERRED` (also counted as `active` on `seqera.workqueue.messages`). | +| `seqera.workqueue.saturated` | Counter | — | — | Polls skipped because the consumer's `ready()` admission gate was closed. | +| `seqera.workqueue.leased` | Gauge | — | entries | Entries currently leased (in-flight), sampled at each renewal tick. Tagged `queue` only. | +| `seqera.workqueue.lease.age.max` | Gauge | — | seconds | Age of the oldest currently-leased entry, sampled at each renewal tick. Tagged `queue` only. | +| `seqera.workqueue.lease.renewal` | Timer | — | seconds | Duration of one lease-renewal tick. Tagged `queue` only. | +| `seqera.workqueue.lease.renewal.errors` | Counter | — | — | Failed renewal round-trips (retried on the next tick). Tagged `queue` only. | +| `seqera.workqueue.lease.renewal.age` | Gauge | — | seconds | Seconds since the last *completed* renewal tick — the stuck-tick detector. Tagged `queue` only. | +| `seqera.workqueue.lease.lost` | Counter | — | — | Leases found owned by another consumer at renewal — the observable residual duplicate window. Tagged `queue` only. | +| `seqera.workqueue.lease.leak` | Counter | — | — | Leases dropped by the age backstop: older than 3× the visibility timeout with no provably-alive owner. Tagged `queue` only. | The `outcome` tag takes one of three values: -- `processed` — the consumer returned `true`; the message was acknowledged and removed from the queue. -- `active` — the consumer returned `false`; the message remains available for redelivery (work still in progress, not a failure). +- `processed` — the consumer decided `ACK`; the message was acknowledged and removed from the queue. +- `active` — the consumer decided `RETRY` or `DEFERRED`; the message remains pending (work still in progress, not a failure). - `errored` — an unhandled exception escaped the consumer or the underlying queue implementation. -Empty receives (no message available) are **ignored** — they do not increment +Empty polls (no message available) are **ignored** — they do not increment `seqera.workqueue.messages_total` and do not contribute to the timer, keeping the timer's `_count`/`_sum`/`_max` aligned with "an entry was processed". @@ -96,7 +168,7 @@ rate(seqera_workqueue_messages_total{outcome="errored"}[1m]) sum by (queue) (rate(seqera_workqueue_messages_total{outcome="errored"}[5m])) / sum by (queue) (rate(seqera_workqueue_messages_total[5m])) -# active-redelivery rate (in-progress receives, not failures) +# active-redelivery rate (in-progress polls, not failures) rate(seqera_workqueue_messages_total{outcome="active"}[1m]) # percentile latencies (server-side aggregation across replicas) @@ -122,7 +194,7 @@ Every metric in the JVM — including these — will then carry an `application` ## Usage -Work distribution with competing consumers and message acknowledgment: +Work distribution with consumer groups and message acknowledgment: ```groovy @Inject @@ -142,82 +214,14 @@ workQueue.offer("user-activity", event) // Consume events class ActivityConsumer implements MessageConsumer { @Override - boolean accept(ActivityEvent event) { + MessageConsumer.Decision accept(ActivityEvent event, MessageLease lease) { analyticsService.recordActivity(event) - return true // Acknowledge message + return MessageConsumer.Decision.ACK // Acknowledge message } } -// Register the consumer; the queue dispatches messages to it asynchronously -workQueue.addConsumer("user-activity", new ActivityConsumer()) -``` - -## Architecture - -`AbstractWorkQueue` runs handlers **asynchronously and concurrently** while -guaranteeing that a given message is processed by exactly one *live* consumer at a -time. A message is owned by its consumer for as long as the handler keeps working — -independent of how long that takes — and ownership is relinquished only when the work -finishes or the consumer dies. - +workQueue.consume("user-activity", new ActivityConsumer()) ``` - offer(msg) ┌──────────────────────────────┐ - │ │ AbstractWorkQueue │ - ▼ │ │ - ┌──────────┐ receive (XREADGROUP/XAUTOCLAIM) │ dispatcher thread │ - │ Redis │◀─────────────────────────────────┤ • acquire a semaphore slot │ - │ stream │ │ • receive one message │ - │ (PEL, │ renewLease (XCLAIM … JUSTID) │ • hand it to the executor │ - │ group) │◀──────────── heartbeat daemon ────┤ (never runs it inline) │ - │ │ every visibility-timeout/3 │ │ - │ │ ack (XACK + XDEL) │ worker (executor thread) │ - │ │◀──────────── on terminal ─────────┤ accept(msg): │ - └──────────┘ │ ├─ true → ack + free slot │ - ▲ │ └─ false → keep lease, │ - │ reclaimed by a peer only if the owner │ re-run after pollInterval│ - │ dies (heartbeat stops → idle > visibility-timeout) via the re-poll sched │ - └─────────────────────────────────────────────────────────────────────────┘ -``` - -**Three mechanisms:** - -1. **Async dispatch (no head-of-line blocking).** The dispatcher thread never runs a - handler; it hands each message to a worker executor and moves on. Handlers run on the - executor supplied via `withHandlerExecutor(...)` — **mandatory, no default** (Micronaut - consumers inject the `@Named(BLOCKING)` executor). A `Semaphore` sized by - `concurrency()` bounds how many messages are in flight at once (backpressure: excess - messages stay in the queue). - -2. **Heartbeat lease (single live runner + safe long handlers).** While a message is in - flight, a daemon renews its Redis consumer-group entry (`XCLAIM … JUSTID`) every - `visibility-timeout / 3`, pinning its idle time near zero so no peer's `XAUTOCLAIM` can - reclaim it — no matter how long the handler runs. If the owning process dies, the - heartbeat stops, idle time crosses the visibility timeout, and a peer reclaims the message - (real dead-consumer failover). A `max-processing-time` safety valve stops renewing a - single invocation that runs pathologically long (logged as *stalled*), without - interrupting its thread. - -3. **In-process re-poll for not-yet-terminal work.** When a handler returns `false` (work - in progress), the message keeps its lease and the handler is **re-invoked in-process** - after `pollInterval` via a scheduler — Redis is not re-read. This makes the re-poll - cadence independent of `visibility-timeout` (which then governs only failover). The next - invocation is scheduled only after the previous one returns, so a given message is - never processed by two overlapping invocations. - -Delivery is **at-least-once** (a crash/pause beyond `visibility-timeout`, or the -`max-processing-time` valve, can hand a still-running message to a peer), so consumers -must be idempotent. The in-memory `LocalWorkQueue` has no pending-entries list, so it -has no lease/heartbeat (renewLease is a no-op); it still benefits from async, concurrent dispatch. - -### Configuration - -| Knob | Where | Default | Governs | -|---|---|---|---| -| `pollInterval()` | `AbstractWorkQueue` | — (subclass) | Idle backoff **and** in-process re-poll cadence | -| `concurrency()` | `AbstractWorkQueue` | `1` | Max in-flight messages (semaphore ceiling) | -| `getVisibilityTimeout()` | `RedisWorkQueueConfig` | — | Dead-consumer failover window | -| `getHeartbeatInterval()` | `RedisWorkQueueConfig` | `visibility-timeout / 3` | Lease renewal cadence | -| `getMaxProcessingTime()` | `RedisWorkQueueConfig` | `15m` | Upper bound on a single `accept()` before its lease is released | ## Testing diff --git a/lib-data-workqueue/VERSION b/lib-data-workqueue/VERSION index 3eefcb9d..227cea21 100644 --- a/lib-data-workqueue/VERSION +++ b/lib-data-workqueue/VERSION @@ -1 +1 @@ -1.0.0 +2.0.0 diff --git a/lib-data-workqueue/build.gradle b/lib-data-workqueue/build.gradle index fc5fbb40..77d3a077 100644 --- a/lib-data-workqueue/build.gradle +++ b/lib-data-workqueue/build.gradle @@ -44,7 +44,6 @@ dependencies { testImplementation "org.apache.groovy:groovy-nio:4.0.31" testImplementation "org.apache.groovy:groovy-templates:4.0.31" testImplementation "org.apache.groovy:groovy-json:4.0.31" - testImplementation project(':lib-lang') testImplementation project(':lib-random') testImplementation "io.micronaut:micronaut-inject-groovy:${micronautCoreVersion}" testImplementation "io.micronaut.test:micronaut-test-spock:${micronautTestVersion}" diff --git a/lib-data-workqueue/changelog.txt b/lib-data-workqueue/changelog.txt index b7755cf5..aeff605b 100644 --- a/lib-data-workqueue/changelog.txt +++ b/lib-data-workqueue/changelog.txt @@ -1,5 +1,29 @@ # lib-data-workqueue changelog +2.0.0 - 19 Aug 2026 +- BREAKING SPI change over the unpublished 1.0.0. This release replaces the placeholder left + by PR #100 with the lease-based implementation that has been running in the Seqera + scheduler, promoted here as the canonical source. +- WorkQueue: receive()/renewLease()/ack()/release() around a Lease record are replaced by + init()/offer()/consume()/length(); consume(queueId, consumer) returns a + MessageConsumer.Decision. +- MessageConsumer: returns Decision{ACK, RETRY, DEFERRED} instead of a boolean, and receives a + MessageLease alongside the message. A returned DEFERRED transfers lease ownership to a task; + a thrown exception settles as RETRY. Adds the optional ready() admission gate for + backpressure. +- MessageLease (new): the settlement handle - ack(), retry(), retryAfter(Duration) and + bindLiveness(BooleanSupplier). Settlement is idempotent, first call wins. +- AbstractWorkQueue: the handler ExecutorService, semaphore slots, re-poll scheduler and + heartbeat daemon are gone; heartbeat renewal now lives in the Redis backend, driven by + MessageLease.bindLiveness. withHandlerExecutor() is removed. Adds a cooperative shutdown: + awaitQuiescent(Duration) stops the dispatcher claiming new work, close(Duration) waits on a + caller budget, and a second close() does not wait again. +- LocalWorkQueue: backed by DelayQueue instead of LinkedBlockingQueue, so a RETRY settlement is + re-queued after a redelivery delay (workqueue.local.retry-delay, default 1s) rather than + spinning. Mirrors the Redis lease semantics in memory. +- No migration path from 1.0.0, which was published but never consumed in production. +- Dropped the lib-lang test dependency; no test imports io.seqera.lang. + 1.0.0 - 11 Jul 2026 - Initial release. This module is the redis-free split of lib-data-stream-redis 1.6.0, renamed to reflect the reliable work-queue semantics it actually implements (competing diff --git a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/AbstractWorkQueue.java b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/AbstractWorkQueue.java index 760f0fc3..b4783a2a 100644 --- a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/AbstractWorkQueue.java +++ b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/AbstractWorkQueue.java @@ -20,15 +20,7 @@ import java.io.Closeable; import java.time.Duration; import java.util.Map; -import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import io.micronaut.core.annotation.Nullable; @@ -45,7 +37,7 @@ * Abstract base implementation of a work queue that provides asynchronous message consumption. * *

This class implements the core functionality for a work queue that continuously consumes - * messages from an underlying queue and delivers them to registered consumers. It provides:

+ * messages from underlying queues and delivers them to registered consumers. It provides:

* *
    *
  • Asynchronous Processing: Uses a background thread to continuously poll for messages
    • @@ -78,13 +70,10 @@ * // Usage * MyWorkQueue queue = new MyWorkQueue(underlyingQueue); * - * // Supply the handler executor (mandatory, no default) before adding consumers - * queue.withHandlerExecutor(executorService); - * * // Add consumer for a specific queue - * queue.addConsumer("user-events", event -> { + * queue.addConsumer("user-events", (event, lease) -> { * processUserEvent(event); - * return true; // Acknowledge successful processing + * return MessageConsumer.Decision.ACK; // Acknowledge successful processing * }); * * // Send messages (will be processed asynchronously by registered consumers) @@ -112,6 +101,12 @@ public abstract class AbstractWorkQueue implements Closeable { private static final AtomicInteger count = new AtomicInteger(); + /** + * Granularity at which an in-loop pause re-checks {@link #closing}, so a cooperative + * shutdown is not held up for a whole poll interval or backoff delay. + */ + private static final long PAUSE_SLICE_MILLIS = 50; + private final Map> listeners = new ConcurrentHashMap<>(); private final ExponentialAttempt attempt = new ExponentialAttempt(); @@ -124,61 +119,22 @@ public abstract class AbstractWorkQueue implements Closeable { private volatile Thread thread; - private final String name0; - /** - * A message picked up from a queue and held while it is processed. The - * {@code queueId} + {@code leaseId} pair identifies the delivered entry; the - * {@code message} is kept so a not-yet-terminal command can be re-invoked in-process - * (Model B) without re-reading it from the queue. + * Set by {@link #awaitQuiescent(Duration)} to stop the dispatcher from claiming further + * messages. The dispatcher observes it at the head of its loop and at every pause slice, + * so it exits at a safe point rather than being interrupted mid-call. */ - private record InFlight(String queueId, String leaseId, String message) { - String key() { - return queueId + '|' + leaseId; - } - } + private volatile boolean closing; /** - * Leases held from pickup to terminal/crash; every entry is heartbeated by the - * daemon so an alive consumer is never reclaimed. Keyed by {@code queueId|leaseId}. + * Set once {@link #close(Duration)} has run its cooperative wait. A second close — the + * {@code @PreDestroy} backstop after an explicit drain, for instance — must not wait again: + * the first call either saw the dispatcher stop or already decided to give up, and repeating + * the wait during bean destruction spends a shutdown budget that was spent once already. */ - private final Map inFlight = new ConcurrentHashMap<>(); + private volatile boolean closeAttempted; - /** - * Subset of {@link #inFlight} whose {@code accept()} invocation is running right now, - * mapped to the wall-clock millis at which that invocation started. Used by the - * heartbeat daemon to enforce {@code max-processing-time} on a single invocation. - */ - private final Map active = new ConcurrentHashMap<>(); - - /** - * Executor that runs the message handlers. Supplied by the consumer via - * {@link #withHandlerExecutor} before the first {@link #addConsumer} — there is no default, - * so {@link #startProcessing()} fails fast if it was never set. Micronaut consumers pass the - * injected {@code BLOCKING} executor. Handler concurrency is bounded by {@link #slots}, not by - * this executor, so it is never sized or shut down here. - */ - private volatile ExecutorService pool; - - /** - * Gates new intake: a permit is acquired when a lease is picked up and held for the - * whole lease lifetime (across re-polls), released on terminal ack / eviction / - * release. This bounds concurrent handlers to {@link #concurrency()} and reserves - * capacity so in-flight commands' re-polls are never starved by new intake. - */ - private volatile Semaphore slots; - - /** - * Schedules delayed re-poll re-submissions for not-yet-terminal commands (Model B). - */ - private volatile ScheduledExecutorService scheduler; - - /** - * Renews every in-flight lease on a fixed cadence so an alive consumer keeps ownership. - */ - private volatile ScheduledExecutorService heartbeat; - - private volatile boolean closed; + private final String name0; /** * Constructs a new queue without metrics instrumentation. Behavior is identical @@ -218,51 +174,17 @@ protected Thread createListenerThread() { } /** - * @return The name of the work queue implementation + * @return The name of the message queue implementation */ protected abstract String name(); /** * @return * The time interval to await before trying to read again the queue - * when no more entries are available. Also the cadence at which a - * not-yet-terminal command is re-invoked in-process (Model B). + * when no more entries are available. */ protected abstract Duration pollInterval(); - /** - * @return - * The maximum number of message handlers that may run concurrently on this - * instance (the worker pool size). Defaults to {@code 1}; subclasses may - * override to enable parallel processing. - */ - protected int concurrency() { - return 1; - } - - /** - * @return - * How often in-flight leases are renewed so an alive consumer keeps ownership - * of its message regardless of how long its handler runs. Must be shorter than - * the underlying queue's visibility timeout; subclasses backed by a configuration - * should wire this to {@code visibility-timeout / 3}. - */ - protected Duration heartbeatInterval() { - final Duration d = queue.heartbeatInterval(); - return d != null ? d : Duration.ofSeconds(20); - } - - /** - * @return - * The upper bound on a single {@code accept()} invocation before its lease is - * released (safety valve); it does not interrupt the handler thread. Defaults - * to {@code 15m}. - */ - protected Duration maxProcessingTime() { - final Duration d = queue.maxProcessingTime(); - return d != null ? d : Duration.ofMinutes(15); - } - /** * Adds a message to the specified queue for asynchronous processing. * @@ -293,14 +215,17 @@ public void offer(String queueId, M message) { *

    Consumer requirements:

    *
      *
    • Must be thread-safe as it may be called from a background thread
      • - *
    • Should return {@code true} to acknowledge successful message processing
      • - *
    • Should return {@code false} if message processing fails or should be retried
      • - *
    • Should handle exceptions gracefully to avoid disrupting queue processing
      • + *
    • Should return {@link MessageConsumer.Decision#ACK} to acknowledge successful processing
      • + *
    • Should return {@link MessageConsumer.Decision#RETRY} if processing should be retried
      • + *
    • Should return {@link MessageConsumer.Decision#DEFERRED} when a task takes the + * message lease and settles it later via {@link MessageLease}
      • + *
    • May override {@link MessageConsumer#ready()} to gate admission — the dispatcher + * skips the queue while the consumer reports not ready
      • *
    * * @param queueId the unique identifier of the queue to consume from; must not be null or empty * @param consumer the message consumer that will process messages; must not be null - * @see MessageConsumer#accept(Object) + * @see MessageConsumer#accept(Object, MessageLease) */ public void addConsumer(String queueId, MessageConsumer consumer) { // the use of synchronized block is meant to prevent a race condition while @@ -317,322 +242,228 @@ public void addConsumer(String queueId, MessageConsumer consumer) { listeners.put(queueId, consumer); // bind the backlog gauge for this queue id (no-op when metrics disabled) metrics.bindBacklog(queueId, () -> queue.length(queueId)); - // finally start the dispatcher thread and its supporting executors + // finally start the listener thread if (thread == null) { - startProcessing(); + thread = createListenerThread(); } } } - /** - * Lazily create the worker pool, the re-poll scheduler, the heartbeat daemon and the - * capacity gate, then start the dispatcher thread. Invoked once, when the first - * consumer is registered. - */ - private void startProcessing() { - // a handler executor must be supplied via withHandlerExecutor() before processing starts - Objects.requireNonNull(pool, "Handler executor not set - call withHandlerExecutor() before addConsumer()"); - // 'slots' — not the executor — bounds how many commands may be in flight at once; - // the cap is a memory/heartbeat ceiling, independent of the executor's threading model. - this.slots = new Semaphore(Math.max(1, concurrency())); - this.scheduler = new ScheduledThreadPoolExecutor(1, daemonFactory(name() + "-repoll-" + count.get())); - this.heartbeat = new ScheduledThreadPoolExecutor(1, daemonFactory(name() + "-heartbeat-" + count.get())); - final long hb = heartbeatInterval().toMillis(); - this.heartbeat.scheduleAtFixedRate(this::heartbeatTick, hb, hb, TimeUnit.MILLISECONDS); - this.thread = createListenerThread(); - } - - /** - * Supply the executor used to run message handlers. Consumers must call this - * before the first {@link #addConsumer} — there is no default executor. - * Micronaut-managed consumers pass the injected {@code @Named(TaskExecutors.BLOCKING)} - * {@link ExecutorService}. The executor is never shut down by {@link #close()} - * (it is shared / container-managed). - * - * @param executor the shared handler executor; must not be {@code null} - */ - public void withHandlerExecutor(ExecutorService executor) { - this.pool = Objects.requireNonNull(executor, "Handler executor cannot be null"); - } - - private static ThreadFactory daemonFactory(String prefix) { - final AtomicInteger seq = new AtomicInteger(); - return runnable -> { - final Thread t = new Thread(runnable, prefix + "-" + seq.getAndIncrement()); - t.setDaemon(true); - return t; - }; - } - /** * Deserialize the message as string into the target message object and process it by applying * the given consumer {@link MessageConsumer}. * * @param msg * The message serialised as a string value + * @param lease + * The {@link MessageLease} settlement handle for this delivery * @param consumer * The consumer {@link MessageConsumer} that will handle the message as a object + * @param count + * An {@link AtomicInteger} counter incremented by one when this method is invoked, + * irrespective if the consumer is successful or not. * @return - * The result of the consumer {@link MessageConsumer} operation. + * The {@link MessageConsumer.Decision} of the consumer operation. */ - protected boolean processMessage(String msg, MessageConsumer consumer) { + protected MessageConsumer.Decision processMessage(String msg, MessageLease lease, MessageConsumer consumer, AtomicInteger count) { + count.incrementAndGet(); final M decoded = encoder.decode(msg); log.trace("Work queue - receiving message={}; decoded={}", msg, decoded); - return consumer.accept(decoded); + return consumer.accept(decoded, lease); + } + + /** + * Run one consume cycle for the given queue and record the outcome on the + * {@link QueueMetrics} handle. The outcome is derived from the {@code count} + * delta (was the consumer lambda invoked?) and the {@link MessageConsumer.Decision} + * returned by {@link WorkQueue#consume}: {@code ACK} records processed, + * {@code RETRY} records active, and {@code DEFERRED} records active plus a + * distinct deferred counter (the task-settled outcome is not timed here). + */ + private MessageConsumer.Decision consumeOne(String queueId, MessageConsumer consumer, AtomicInteger count) { + final long sample = metrics.startSample(); + final int countBefore = count.get(); + Outcome outcome = Outcome.EMPTY; + try { + final MessageConsumer.Decision decision = queue.consume(queueId, + (String msg, MessageLease lease) -> processMessage(msg, lease, consumer, count)); + if (count.get() != countBefore) { + outcome = decision == MessageConsumer.Decision.ACK ? Outcome.PROCESSED : Outcome.ACTIVE; + if (decision == MessageConsumer.Decision.DEFERRED) { + metrics.deferred(queueId); + } + } + return decision; + } + catch (Throwable t) { + outcome = Outcome.ERRORED; + throw t; + } + finally { + metrics.recordOutcome(sample, queueId, outcome); + } } /** - * The dispatcher loop (runs on the listener thread). It never runs a handler itself: - * for every queue that has free pool capacity it polls one message (without acking) - * and submits its processing to the worker pool, then sleeps for {@link #pollInterval()} - * when nothing was polled this cycle. + * Process the messages as they are available from the underlying queue */ protected void processMessages() { - log.trace("Work queue - starting dispatcher thread"); - while (!Thread.currentThread().isInterrupted()) { + log.trace("Work queue - starting listener thread"); + // `closing` is checked first so a cooperative shutdown claims no further message; the + // cycle already in progress below always runs to completion, which is what lets a + // consumer finish its work (and its database writes) before the context tears down. + while (!closing && !Thread.currentThread().isInterrupted()) { try { - boolean polled = false; + final var count = new AtomicInteger(); + boolean progressed = false; for (Map.Entry> entry : listeners.entrySet()) { - // poll a queue only when a worker slot is free (backpressure); the - // permit is held for the whole lease lifetime so re-polls of in-flight - // commands are never starved by new intake - if (!slots.tryAcquire()) { - break; + final var queueId = entry.getKey(); + final var consumer = entry.getValue(); + // admission gate: do not claim from a queue whose consumer is not + // ready — counts as no-message, so an idle loop still pauses below + if (!consumer.ready()) { + metrics.saturated(queueId); + continue; } - // dispatchOne releases the permit itself when nothing is polled - polled = dispatchOne(entry.getKey()) || polled; + final MessageConsumer.Decision decision = consumeOne(queueId, consumer, count); + // only ACK and DEFERRED are progress: a RETRY must NOT keep the loop + // hot — a consumer retrying fast (e.g. against the local queue, which + // has no claim clock) would otherwise redeliver at loop speed + progressed |= decision == MessageConsumer.Decision.ACK + || decision == MessageConsumer.Decision.DEFERRED; } // reset the attempt count because no error has been thrown attempt.reset(); - // if nothing was polled this cycle, sleep for a while before retrying - if (!polled) { + // pause unless a cycle made real progress, so idle AND retry-only cycles + // are both paced by the poll interval + if (!progressed) { log.trace("Work queue - await before checking for new messages"); - Thread.sleep(pollInterval().toMillis()); + pause(pollInterval().toMillis()); } } - catch (InterruptedException e) { - log.debug("Work queue interrupt exception - cause: {}", e.getMessage()); - Thread.currentThread().interrupt(); - break; - } catch (Throwable e) { + // A forced stop (close() fallback) surfaces as an interrupt, possibly wrapped by + // the underlying client. Treat it as "exit now", not as a queue error to retry: + // logging it at ERROR with a backoff would turn every hard shutdown into noise. + if (e instanceof InterruptedException || Thread.currentThread().isInterrupted()) { + log.debug("Work queue {} interrupted - exiting listener thread", name0); + Thread.currentThread().interrupt(); + break; + } final var d0 = attempt.delay(); log.error("Unexpected error on work queue {} (await: {}) - cause: {}", name0, d0, e.getMessage(), e); - sleep(d0.toMillis()); - } - } - log.trace("Work queue - exiting dispatcher thread"); - } - - /** - * Poll a single queue (a worker permit has already been acquired by the caller) and, - * if a message is available, register it as in-flight and submit it to the pool. - * If nothing is available the permit is released and {@code false} is returned. - * - * @return {@code true} if a message was polled and submitted, {@code false} otherwise - */ - private boolean dispatchOne(String queueId) { - boolean submitted = false; - try { - final WorkQueue.Lease lease = queue.receive(queueId); - if (lease == null) { - metrics.recordOutcome(metrics.startSample(), queueId, Outcome.EMPTY); - return false; - } - final var e = new InFlight(queueId, lease.id(), lease.message()); - // Guard against self-reclaim: if the heartbeat falls behind by more than the - // visibility timeout, this instance's own receive() (XAUTOCLAIM) can re-deliver an - // entry it is already processing. The reclaim only refreshed the lease idle time, so - // keep the live in-flight entry and drop the duplicate — otherwise a second handler - // runs concurrently and its permit leaks (the original remove() returns null). - if (inFlight.putIfAbsent(e.key(), e) != null) { - return false; // 'submitted' stays false → finally releases this permit - } - submitRun(e); - submitted = true; - return true; - } - finally { - // the permit is held only once the lease is in flight; release it on an empty - // poll or an exception so the single acquire in the dispatcher stays balanced - if (!submitted) { - slots.release(); + pause(d0.toMillis()); } } + log.trace("Work queue - exiting listener thread"); } /** - * Submit the processing of an in-flight lease to the worker pool. Swallows the - * rejection that occurs when the pool is being shut down. + * Sleep up to {@code millis}, returning early once {@link #closing} is set or the thread is + * interrupted. Used instead of a single long sleep so neither the poll interval nor an + * exponential backoff delay can hold up a cooperative shutdown. */ - private void submitRun(InFlight e) { - try { - pool.execute(() -> run(e)); - } - catch (RejectedExecutionException ex) { - log.debug("Work queue - worker pool rejected task for entry={} (shutting down)", e.key()); + private void pause(long millis) { + final long deadline = System.currentTimeMillis() + millis; + long remaining; + while (!closing + && !Thread.currentThread().isInterrupted() + && (remaining = deadline - System.currentTimeMillis()) > 0) { + sleep(Math.min(PAUSE_SLICE_MILLIS, remaining)); } } /** - * Runs a single {@code accept()} invocation on a worker thread. On {@code true} - * (terminal) it acks the message and drops the lease; on {@code false} (Model B, - * not-yet-terminal) it keeps the lease in-flight and schedules the next invocation - * after {@link #pollInterval()} — strictly serial per command, since the next - * invocation is scheduled only after this one returned. - */ - private void run(InFlight e) { - final boolean accepted = invokeHandler(e); - if (accepted) { - acknowledge(e); - } - else if (shouldRepoll(e)) { - scheduleRepoll(e); - } - } - - /** - * Run one {@code accept()} invocation on the worker thread, recording the metrics - * outcome. Returns {@code true} for a terminal result, {@code false} for - * not-yet-terminal or an error (both keep the lease for a later re-poll). + * Stop claiming new messages and wait for the dispatcher to finish the cycle it is running. + * + *

    This is the cooperative half of {@link #close()}, exposed separately so a caller can + * drain the queue while its collaborators — a database connection pool, for instance — are + * still usable, and only then release resources. + * + *

    Safe to call more than once, and safe to call before any consumer was registered. + * + * @param timeout + * how long to wait for the dispatcher to exit + * @return + * {@code true} if the dispatcher stopped within the timeout, {@code false} if it is + * still running, in which case the caller decides whether to force a stop */ - private boolean invokeHandler(InFlight e) { - final MessageConsumer consumer = listeners.get(e.queueId()); - final long sample = metrics.startSample(); - boolean accepted = false; - Outcome outcome = Outcome.ACTIVE; - active.put(e.key(), System.currentTimeMillis()); - try { - accepted = processMessage(e.message(), consumer); - outcome = accepted ? Outcome.PROCESSED : Outcome.ACTIVE; - } - catch (Throwable t) { - outcome = Outcome.ERRORED; - log.error("Work queue - error processing entry={} - cause: {}", e.key(), t.getMessage(), t); - } - finally { - active.remove(e.key()); - metrics.recordOutcome(sample, e.queueId(), outcome); + public boolean awaitQuiescent(Duration timeout) { + closing = true; + final Thread t = thread; + if (t == null) { + return true; } - return accepted; - } - - /** Terminal result: acknowledge the message and release its lease. */ - private void acknowledge(InFlight e) { try { - queue.ack(e.queueId(), e.leaseId()); + t.join(Math.max(1, timeout.toMillis())); } - catch (Throwable t) { - log.error("Work queue - error acking entry={} - cause: {}", e.key(), t.getMessage(), t); - } - finally { - releaseLease(e.key()); + catch (InterruptedException e) { + log.info("Work queue {} interrupted while awaiting quiescence", name0, e); + Thread.currentThread().interrupt(); } - } - - /** Whether a not-yet-terminal command should be re-polled: still owned and not shutting down. */ - private boolean shouldRepoll(InFlight e) { - return !closed && inFlight.containsKey(e.key()); + return !t.isAlive(); } /** - * Keep the lease (the heartbeat keeps renewing it, so no reclaim/migration) and schedule - * the next in-process invocation after {@link #pollInterval()} — strictly serial, since - * it is scheduled only after the previous invocation returned. + * Shutdown orderly the queue. + * + *

    Cooperative first: {@link #awaitQuiescent(Duration)} lets the dispatcher finish the + * message it is holding and leave the loop at a safe point. Interrupting a thread parked in a + * Redis read can hand a RESP-desynced connection back to the pool (libseqera#92), so the + * dispatcher is never interrupted: the flag alone guarantees it exits, and it is a daemon. + * + *

    Uses {@link #closeTimeout()} as the budget. Callers that have already spent part of an + * overall shutdown budget should call {@link #close(Duration)} with what remains; callers that + * need the drain to complete while other beans are still alive should call + * {@link #awaitQuiescent(Duration)} themselves, ahead of either. */ - private void scheduleRepoll(InFlight e) { - try { - scheduler.schedule(() -> submitRun(e), pollInterval().toMillis(), TimeUnit.MILLISECONDS); - } - catch (RejectedExecutionException ex) { - log.debug("Work queue - re-poll scheduler rejected entry={} (shutting down)", e.key()); - } + @Override + public void close() { + close(closeTimeout()); } /** - * Drop a lease from the in-flight set and free its capacity permit. This pair is the - * single invariant "a permit is held iff its key is in-flight"; returns {@code true} if - * this call performed the removal (so callers can log only a real eviction). + * Shutdown orderly the queue within an explicit budget. + * + *

    Exists so a caller that has already spent part of an overall shutdown budget can pass what + * remains, instead of this method starting a second, independent timer. A caller that drains + * first and then closes with {@link #closeTimeout()} can otherwise overrun its own deadline — + * and if that deadline came from a container's graceful-shutdown grace period, overrunning it + * means being hard-stopped mid-drain, which is the opposite of what draining is for. + * + *

    Only the first close waits. Repeated calls — an explicit drain followed by the + * {@code @PreDestroy} backstop — return immediately, for the same budget reason. + * + * @param timeout how long to wait for a cooperative stop before interrupting the dispatcher */ - private boolean releaseLease(String key) { - if (inFlight.remove(key) != null) { - slots.release(); - return true; + public void close(Duration timeout) { + if (thread == null) { + return; } - return false; - } - - /** - * Heartbeat tick: renew every in-flight lease so an alive consumer keeps ownership, - * and release the lease of any single invocation that has exceeded - * {@link #maxProcessingTime()} (safety valve; does not interrupt the handler thread). - */ - private void heartbeatTick() { - final long now = System.currentTimeMillis(); - final long maxMillis = maxProcessingTime().toMillis(); - for (InFlight e : inFlight.values()) { - final String key = e.key(); - final long start = active.getOrDefault(key, now); - if (now - start > maxMillis) { - // a single invocation is stalled beyond the bound: stop renewing so the - // lease becomes reclaimable, and free its capacity permit - if (releaseLease(key)) { - log.warn("Work queue - releasing lease of stalled entry={} after {} - reclaimable after visibility timeout", - key, Duration.ofMillis(now - start)); - } - } - else { - try { - queue.renewLease(e.queueId(), e.leaseId()); - } - catch (Throwable t) { - // swallow transient errors; the next tick retries - log.warn("Work queue - error renewing lease for entry={} - cause: {}", key, t.getMessage()); - } - } + if (closeAttempted) { + return; + } + closeAttempted = true; + if (awaitQuiescent(timeout)) { + return; } + // Stop waiting, but do not interrupt. The `closing` flag already guarantees the dispatcher + // exits at its next loop-head check, and the thread is a daemon so it can never hold up JVM + // exit — so an interrupt only shortens a wait we have already decided not to keep making. + // Against that it is actively harmful: it can hand a RESP-desynced connection back to the + // pool (libseqera#92), and it was observed to propagate into a handler still running on the + // executor, cutting short the very work a drain exists to protect. + log.warn("Work queue {} still running after {} - leaving it to exit on its own", name0, timeout); } /** - * Shutdown orderly the queue: stop the dispatcher, cancel pending re-polls, drain - * the worker pool so active handlers finish and ack, release any remaining leases so - * they are redelivered, and finally stop the heartbeat daemon. + * How long {@link #close()} waits for a cooperative stop before interrupting the dispatcher. + * Subclasses may override to align with an application-level shutdown budget. + * + * @return the cooperative close timeout, {@code 10s} by default */ - @Override - public void close() { - if (thread == null) { - return; - } - closed = true; - // 1. stop the dispatcher - thread.interrupt(); - try { - thread.join(1_000); - } - catch (Exception e) { - log.debug("Unexpected error while terminating {} - cause: {}", name0, e.getMessage()); - } - // 2. cancel pending scheduled re-polls - if (scheduler != null) { - scheduler.shutdownNow(); - } - // 3. the handler executor is shared / container-managed — not shut down here; - // any active handler finishes on its own (short-lived) and acks - // 4. release any lease still held so it is redelivered without waiting for lapse - for (InFlight e : inFlight.values()) { - if (inFlight.remove(e.key()) != null) { - try { - queue.release(e.queueId(), e.leaseId()); - } - catch (Throwable t) { - log.debug("Work queue - error releasing entry={} on shutdown - cause: {}", e.key(), t.getMessage()); - } - } - } - // 5. stop the heartbeat daemon last (any remaining leases lapse -> peers reclaim) - if (heartbeat != null) { - heartbeat.shutdownNow(); - } + protected Duration closeTimeout() { + return Duration.ofSeconds(10); } public int length(String queueId) { diff --git a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/LocalWorkQueue.java b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/LocalWorkQueue.java index 5c1ea54c..a20e2617 100644 --- a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/LocalWorkQueue.java +++ b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/LocalWorkQueue.java @@ -17,17 +17,25 @@ package io.seqera.data.workqueue; +import java.time.Duration; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.DelayQueue; +import java.util.concurrent.Delayed; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import io.micronaut.context.annotation.Requires; +import io.micronaut.context.annotation.Value; import io.seqera.activator.redis.RedisActivator; +import io.seqera.data.workqueue.MessageConsumer; +import io.seqera.data.workqueue.MessageLease; +import io.seqera.data.workqueue.WorkQueue; import jakarta.inject.Singleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * In-memory implementation of {@link WorkQueue} using Java {@link LinkedBlockingQueue} + * In-memory implementation of {@link WorkQueue} using Java {@link DelayQueue} * as the underlying storage mechanism. This implementation is designed exclusively for * development, testing, and local environments. * @@ -40,9 +48,18 @@ *

  • Local Only: Messages exist only within the current JVM instance
    • *
  • No Persistence: All messages are lost when the application stops
    • *
  • No Distribution: Cannot share messages across multiple application instances
    • - *
  • Simple Queuing: Messages are processed in FIFO order using blocking queues
    • + *
  • Paced Retries: a {@code RETRY} settlement re-queues the message with a + * redelivery delay ({@code workqueue.local.retry-delay}, default 1s) — the local analog + * of the pacing the visibility timeout provides on Redis. Without it, a consumer that + * retries fast (a handler repeatedly declaring RUNNING, or throwing quickly) would + * drive a hot loop of continuous re-deliveries in non-Redis deployments
    • *
* + *

Lease semantics mirror the Redis implementation in-memory: a message whose consumer + * returned {@link MessageConsumer.Decision#DEFERRED} stays unavailable until its + * {@link MessageLease} settles — {@code ack()} removes it, {@code retry()} makes it + * redeliverable after the retry delay. Settlement is idempotent, first call wins. + * *

This implementation automatically activates when the 'redis' environment is not * active, making it ideal for: *

    @@ -52,7 +69,7 @@ *
* *

Each queue is backed by its own {@link ConcurrentHashMap} entry containing - * a {@link LinkedBlockingQueue} for thread-safe message handling. + * a {@link DelayQueue} for thread-safe, availability-aware message handling. * * @author Paolo Di Tommaso * @since 1.0 @@ -63,14 +80,23 @@ public class LocalWorkQueue implements WorkQueue { private static final Logger log = LoggerFactory.getLogger(LocalWorkQueue.class); - private final ConcurrentHashMap> delegate = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> delegate = new ConcurrentHashMap<>(); + + /** + * Redelivery delay applied by a {@code RETRY} settlement — the local analog of the + * visibility-timeout cadence on Redis, deliberately much shorter: local is a dev/test + * profile where snappy re-polls are a feature; what matters is that the cadence is + * bounded, not that it matches production. + */ + @Value("${workqueue.local.retry-delay:1s}") + private Duration retryDelay = Duration.ofSeconds(1); /** * {@inheritDoc} */ @Override public void init(String queueId) { - delegate.put(queueId, new LinkedBlockingQueue<>()); + delegate.put(queueId, new DelayQueue<>()); } /** @@ -78,65 +104,138 @@ public void init(String queueId) { */ @Override public void offer(String queueId, String message) { + offer(queueId, message, Duration.ZERO); + } + + private void offer(String queueId, String message, Duration delay) { delegate .get(queueId) - .offer(message); + .offer(new DelayedMessage(message, delay)); } /** * {@inheritDoc} * - *

Reads one message off the local queue. There is no pending-entries list, so - * the lease id is simply the message value itself (used to re-offer it on release). + *

The polled message is naturally "leased" by being out of the queue: an + * {@code ACK} drops it, a {@code RETRY} — returned, settled via the lease, or caused + * by a consumer throw — re-queues it with the retry delay, and a {@code DEFERRED} + * leaves it out of the queue until the lease settles. */ @Override - public Lease receive(String queueId) { - final var message = delegate + public MessageConsumer.Decision consume(String queueId, MessageConsumer consumer) { + // DelayQueue.poll() only returns a message whose availability delay has expired + final var delayed = delegate .get(queueId) .poll(); - if (message == null) { + if (delayed == null) { return null; } - return new Lease<>(message, message); + + final String message = delayed.message; + final LocalLease lease = new LocalLease(queueId, message); + final MessageConsumer.Decision decision; + try { + decision = consumer.accept(message, lease); + } + catch (Throwable e) { + // consumer throw settles as RETRY: the message redelivers after the retry delay + log.debug("Failed to consume message from queue={} - cause: {}", queueId, e.getMessage(), e); + lease.retry(); + return MessageConsumer.Decision.RETRY; + } + settle(lease, decision); + return decision; } /** - * {@inheritDoc} - * - *

No pending-entries list ⇒ no lease semantics ⇒ no-op. + * Settle a synchronous decision; first settlement wins, so a consumer that already + * settled through the lease makes the returned decision a no-op. */ - @Override - public void renewLease(String queueId, String leaseId) { - // no-op: the local queue has no pending-entries list + private void settle(LocalLease lease, MessageConsumer.Decision decision) { + if (decision == null || decision == MessageConsumer.Decision.RETRY) { + lease.retry(); + return; + } + if (decision == MessageConsumer.Decision.ACK) { + lease.ack(); + } + // DEFERRED: the consumer's task owns the lease and settles it later } /** * {@inheritDoc} - * - *

The message was already removed from the queue on {@link #receive(String)}, - * so acknowledgment is a no-op (the entry is simply dropped). */ @Override - public void ack(String queueId, String leaseId) { - // no-op: the entry was removed from the queue on receive + public int length(String queueId) { + return delegate.get(queueId).size(); } /** - * {@inheritDoc} - * - *

Re-offers the message onto the queue so it is redelivered later, mimicking the - * behavior of a Redis stream pending entry that is not acknowledged. + * A queued message with an availability time: {@link DelayQueue} hands it out only + * once the delay expires. Fresh offers carry a zero delay (immediately available); + * retries carry the retry delay. Sequential {@link System#nanoTime()} stamps keep + * FIFO order among equally-available messages. */ - @Override - public void release(String queueId, String leaseId) { - offer(queueId, leaseId); + private static final class DelayedMessage implements Delayed { + + private final String message; + + private final long availableAt; + + private DelayedMessage(String message, Duration delay) { + this.message = message; + this.availableAt = System.nanoTime() + delay.toNanos(); + } + + @Override + public long getDelay(TimeUnit unit) { + return unit.convert(availableAt - System.nanoTime(), TimeUnit.NANOSECONDS); + } + + @Override + public int compareTo(Delayed other) { + if (other instanceof DelayedMessage dm) { + return Long.compare(availableAt, dm.availableAt); + } + return Long.compare(getDelay(TimeUnit.NANOSECONDS), other.getDelay(TimeUnit.NANOSECONDS)); + } } /** - * {@inheritDoc} + * In-memory settlement handle: the message is already out of the queue, so + * {@code ack()} only marks it settled while {@code retry()} re-queues it with the + * retry delay. */ - @Override - public int length(String queueId) { - return delegate.get(queueId).size(); + private final class LocalLease implements MessageLease { + + private final String queueId; + + private final String message; + + private final AtomicBoolean settled = new AtomicBoolean(); + + private LocalLease(String queueId, String message) { + this.queueId = queueId; + this.message = message; + } + + @Override + public void ack() { + settled.compareAndSet(false, true); + } + + @Override + public void retry() { + if (settled.compareAndSet(false, true)) { + offer(queueId, message, retryDelay); + } + } + + @Override + public void retryAfter(Duration delay) { + if (settled.compareAndSet(false, true)) { + offer(queueId, message, delay); + } + } } } diff --git a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/MessageConsumer.java b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/MessageConsumer.java index c34b3bf8..fbeb6ea2 100644 --- a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/MessageConsumer.java +++ b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/MessageConsumer.java @@ -21,113 +21,78 @@ * Interface for consuming messages from a work queue. * *

A message consumer defines how individual messages should be processed when they - * are read from a queue. The consumer's return value determines whether the message - * was successfully processed and should be acknowledged, or if it failed and may need - * to be reprocessed.

+ * are read from a queue. The consumer returns a {@link Decision} that determines how + * the message settles: acknowledged and removed, left pending for redelivery, or + * deferred to a task that settles it later through the {@link MessageLease} handle.

* - *

Key characteristics:

+ *

Decision semantics:

*
    - *
  • Single Message Processing: Each invocation processes exactly one message
    • - *
  • Acknowledgment Control: Return value controls message acknowledgment
    • - *
  • Error Handling: Failed processing can trigger redelivery
    • - *
  • Stateless: Should be stateless and thread-safe when possible
    • + *
  • {@link Decision#ACK}: the message was processed; it is + * acknowledged and removed from the queue immediately.
    • + *
  • {@link Decision#RETRY}: the message was not processed; it is + * left pending and redelivered after the queue's visibility timeout.
    • + *
  • {@link Decision#DEFERRED}: a task now owns the message lease; + * the entry stays leased (heartbeated where the underlying queue supports it) + * until the task settles it via {@link MessageLease#ack()} or + * {@link MessageLease#retry()}. Only a returned {@code DEFERRED} + * transfers the lease — a thrown exception settles as {@code RETRY}.
    • *
* - *

Common implementation patterns:

- * 
{@code
- * // Simple message processor
- * MessageConsumer orderProcessor = order -> {
- *     try {
- *         processOrder(order);
- *         return true; // Success - acknowledge message
- *     } catch (Exception e) {
- *         log.error("Failed to process order", e);
- *         return false; // Failure - don't acknowledge
- *     }
- * };
- *
- * // Conditional processing
- * MessageConsumer notificationFilter = event -> {
- *     if (event.getPriority() == Priority.HIGH) {
- *         sendImmediateNotification(event);
- *         return true; // Processed
- *     }
- *     return false; // Skip - let another consumer handle it
- * };
- *
- * // Batch processing with validation
- * MessageConsumer batchProcessor = record -> {
- *     if (isValidRecord(record)) {
- *         addToBatch(record);
- *         if (batchIsFull()) {
- *             processBatch();
- *         }
- *         return true; // Successfully added to batch
- *     } else {
- *         log.warn("Invalid record: {}", record);
- *         return true; // Acknowledge to prevent reprocessing invalid data
- *     }
- * };
- * }
- * - *

Return value semantics:

- *
    - *
  • {@code true}: Message processed successfully, acknowledge and remove from queue
    • - *
  • {@code false}: Message not processed, leave available for other consumers
    • - *
- * - *

Error handling strategies:

- *
    - *
  • Retry: Return {@code false} to allow reprocessing
    • - *
  • Dead Letter: Return {@code true} after logging to prevent infinite retries
    • - *
  • Circuit Breaker: Temporarily return {@code false} when downstream services are unavailable
    • - *
+ *

The optional {@link #ready()} admission gate lets a consumer signal backpressure: + * the dispatcher does not claim messages from a queue while its consumer reports + * {@code false}.

* * @param the type of messages that this consumer can process * * @author Paolo Di Tommaso * @since 1.0 * @see WorkQueue#consume(String, MessageConsumer) + * @see MessageLease * @see AbstractWorkQueue */ @FunctionalInterface public interface MessageConsumer { + /** + * How a delivered message settles. + */ + enum Decision { + /** Settle now: acknowledge and remove the message from the queue. */ + ACK, + /** Leave pending: the message is redelivered after the visibility timeout. */ + RETRY, + /** A task owns the lease; it will settle via {@link MessageLease}. */ + DEFERRED + } + /** * Processes a single message from a queue. * *

This method is called by the queue infrastructure when a message is available - * for processing. The implementation should handle the message according to its - * business logic and return an appropriate acknowledgment status.

- * - *

Processing guidelines:

- *
    - *
  • Idempotent: Should handle duplicate messages gracefully
    • - *
  • Fast: Avoid long-running operations that block other messages
    • - *
  • Exception Safe: Handle exceptions appropriately, don't let them propagate
    • - *
  • Logging: Log important events for debugging and monitoring
    • - *
- * - *

Return value meaning:

- *
    - *
  • {@code true}: Message was successfully processed and should be acknowledged. - * The message will be marked as consumed and will not be delivered to other consumers.
    • - *
  • {@code false}: Message was not processed successfully or was rejected. - * The message remains available for consumption by other consumers or for retry.
    • - *
+ * for processing. The implementation handles the message according to its business + * logic and returns the {@link Decision} that settles it — or {@link Decision#DEFERRED} + * to transfer the settlement responsibility to a task via the given lease.

* - *

Common scenarios for returning {@code false}:

- *
    - *
  • Temporary downstream service unavailability
    • - *
  • Message doesn't match consumer's processing criteria
    • - *
  • Resource constraints (memory, connections, etc.)
    • - *
  • Backpressure from downstream systems
    • - *
+ *

An exception thrown out of this method settles the message as + * {@link Decision#RETRY}: nothing stays leased, and the message is redelivered on + * the queue's claim cadence.

* * @param message the message to be processed; may be null depending on queue implementation - * @return {@code true} if the message was successfully processed and should be acknowledged, - * {@code false} if the message was not processed and should remain available + * @param lease the settlement handle for this delivery; only relevant when returning + * {@link Decision#DEFERRED}, ignored otherwise + * @return the settlement decision; must not be null + */ + Decision accept(T message, MessageLease lease); + + /** + * Admission gate: the dispatcher does not claim messages from this queue while + * this method returns {@code false}. A skipped queue counts as an empty poll for + * the dispatcher's idle pause. + * + * @return {@code true} when the consumer can accept a new message */ - boolean accept(T message); + default boolean ready() { + return true; + } } diff --git a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/MessageLease.java b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/MessageLease.java new file mode 100644 index 00000000..ce2b425e --- /dev/null +++ b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/MessageLease.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.workqueue; + +import java.time.Duration; +import java.util.function.BooleanSupplier; + +/** + * Handle to settle a message whose consumer returned + * {@link MessageConsumer.Decision#DEFERRED} — the entry stays leased (owned by this + * consumer and heartbeated where the underlying queue supports it) until one of the + * two settlement methods is invoked. + * + *

Settlement contract: + *

    + *
  • Idempotent, first call wins: the first {@link #ack()} or + * {@link #retry()} settles the lease; every later call on either method is a + * no-op.
    • + *
  • Callable from any thread: the task that owns the lease may + * settle it from a thread other than the one that consumed the message.
    • + *
  • A lease must never outlive its task: callers are expected to + * settle on every exit path (a {@code finally}-guarded {@link #retry()} composes + * with a happy-path {@link #ack()} thanks to idempotence).
    • + *
+ * + * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com> + * @see MessageConsumer.Decision#DEFERRED + */ +public interface MessageLease { + + /** + * Settle the message as processed: stop renewing the lease, then acknowledge and + * remove the entry from the queue (best-effort — a failed acknowledgment degrades + * to a redelivery that acknowledges on the caller's terminal check). + */ + void ack(); + + /** + * Settle the message as not processed: stop renewing the lease only. The entry + * stays pending and is redelivered on the queue's claim cadence — no further + * network call is made; stopping renewal is the release. + */ + void retry(); + + /** + * Settle the message as not processed, with a floor on its redelivery: the entry + * is redelivered no earlier than {@code delay} from now. Queues with lease + * renewal keep the entry leased — never stalled — until {@code delay} minus the + * claim cadence has elapsed, then release it to the normal redelivery clock; a + * delay at or below the claim cadence degrades to a plain {@link #retry()}. + * + *

Exists to pace re-polls independently of the failure-detection clock: without + * it, shortening the visibility timeout for faster crash detection silently + * multiplies the polling load on every dependency the re-polls touch. + * + * @param delay the earliest redelivery, measured from now + */ + void retryAfter(Duration delay); + + /** + * Bind a liveness probe for the task that owns this lease. Queues that apply a + * lease-age backstop (dropping leases whose settlement path appears to have never + * run) consult the probe before pruning: a lease whose owner is provably alive is + * never age-pruned, so a legitimately slow task keeps its lease for as long as it + * actually runs. Without a bound probe the age backstop applies unconditionally. + * + *

The probe must be cheap, thread-safe and non-throwing — typically + * {@code () -> !task.isDone()} on the {@link java.util.concurrent.Future} of the + * owning task. The default implementation is a no-op, for queues without lease + * renewal. + * + * @param alive returns {@code true} while the owning task is still running + */ + default void bindLiveness(BooleanSupplier alive) { + } + +} diff --git a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/WorkQueue.java b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/WorkQueue.java index d1d4e63f..d14cf63e 100644 --- a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/WorkQueue.java +++ b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/WorkQueue.java @@ -17,28 +17,26 @@ package io.seqera.data.workqueue; -import java.time.Duration; - /** - * Interface for a distributed, reliable work queue with competing consumers. + * Interface for a distributed work queue that supports real-time event processing. * - *

A work queue provides the following semantics:

+ *

A work queue of this kind differs from a fire-and-forget message queue in several + * key ways:

*
    - *
  • Competing Consumers: Multiple consumers pull work from the same queue, - * but each message is delivered to exactly one live owner at a time
    • - *
  • Acknowledgment: A message is removed only once it is acknowledged; - * otherwise it remains available for redelivery
    • - *
  • Lease / visibility timeout: A delivered message is leased to its owner; - * the lease is kept alive by heartbeat renewal for as long as the handler runs
    • - *
  • Redelivery & dead-owner reclaim: If the owner dies (its lease lapses - * past the visibility timeout) the message is reclaimed by a peer
    • + *
  • Persistent Log: Messages are stored as an append-only log that can be replayed
    • + *
  • Multiple Consumers: Multiple consumers can read from the same queue independently
    • + *
  • Ordered Delivery: Messages are delivered in the order they were added
    • + *
  • Consumer Groups: Consumers can be grouped for load balancing and fault tolerance
    • + *
  • Log Replay: Consumers can start reading from any point in the queue history
    • *
* *

Work queues are ideal for:

*
    - *
  • Task/job distribution across workers
    • - *
  • Reliable command processing with at-least-once delivery
    • - *
  • Background processing with dead-consumer failover
    • + *
  • Event sourcing and audit logging
    • + *
  • Real-time data processing and analytics
    • + *
  • Microservice event communication
    • + *
  • Activity feeds and notification systems
    • + *
  • Change data capture (CDC) systems
    • *
* *

Usage pattern:

@@ -48,21 +46,29 @@ * queue.init("user-events"); * queue.offer("user-events", new UserLoginEvent(userId, timestamp)); * - * // Consume messages - * MessageConsumer consumer = event -> { + * // Consume messages asynchronously + * MessageConsumer consumer = (event, lease) -> { * processEvent(event); - * return true; // Acknowledge successful processing + * return MessageConsumer.Decision.ACK; // Acknowledge successful processing * }; * * while (hasMoreMessages) { - * boolean processed = queue.consume("user-events", consumer); - * if (!processed) { + * MessageConsumer.Decision decision = queue.consume("user-events", consumer); + * if (decision == null) { * // No messages available, wait before trying again * Thread.sleep(pollInterval); * } * } * } * + *

Implementations may provide additional features such as:

+ *
    + *
  • Message partitioning for scalability
    • + *
  • Consumer group management
    • + *
  • Queue retention policies
    • + *
  • Dead letter handling for failed messages
    • + *
+ * * @param the type of messages that can be sent through the queue * * @author Paolo Di Tommaso @@ -95,11 +101,21 @@ public interface WorkQueue { /** * Adds a message to the specified queue. * - *

Messages are appended to the queue in the order they are offered.

+ *

Messages are appended to the queue in the order they are offered, creating + * an immutable, ordered log of events. Once added, messages typically cannot be + * modified or deleted, ensuring data integrity and enabling replay.

* *

This operation is generally atomic and thread-safe, allowing multiple * producers to safely add messages concurrently to the same queue.

* + *

Message properties:

+ *
    + *
  • Ordering: Messages maintain their insertion order
    • + *
  • Durability: Messages are persisted for later consumption
    • + *
  • Uniqueness: Each message receives a unique sequence number or ID
    • + *
  • Timestamp: Messages are typically timestamped upon arrival
    • + *
+ * * @param queueId the unique identifier of the target queue; must not be null or empty * @param message the message to be added to the queue; may be null depending on implementation * @throws IllegalArgumentException if queueId is null or empty @@ -110,122 +126,35 @@ public interface WorkQueue { * Attempts to consume a single message from the queue using the provided consumer. * *

This method attempts to read one message from the queue and pass it to the - * consumer for processing. The method returns {@code true} if a message was - * successfully processed, or {@code false} if no message was available or the - * consumer rejected the message.

+ * consumer for processing, together with a {@link MessageLease} settlement handle. + * The consumer's {@link MessageConsumer.Decision} controls how the message settles.

* *

Message consumption behavior:

*
    *
  • Non-blocking: Returns immediately if no messages are available
    • + *
  • Ordered: Messages are delivered in queue order
    • *
  • At-least-once: Messages may be delivered multiple times in failure scenarios
    • - *
  • Consumer Control: Consumer return value determines acknowledgment
    • + *
  • Consumer Control: Consumer decision determines settlement
    • *
* - *

Consumer acknowledgment:

+ *

Settlement semantics:

*
    - *
  • Return {@code true} to acknowledge successful processing
    • - *
  • Return {@code false} to indicate processing failure or rejection
    • - *
  • Unacknowledged messages may be redelivered to other consumers
    • + *
  • {@link MessageConsumer.Decision#ACK} — the message is acknowledged and removed
    • + *
  • {@link MessageConsumer.Decision#RETRY} — the message stays pending and is + * redelivered after the visibility timeout
    • + *
  • {@link MessageConsumer.Decision#DEFERRED} — the message stays leased until + * the consumer's task settles it via {@link MessageLease}
    • + *
  • An exception thrown by the consumer settles the message as {@code RETRY} + * and propagates to the caller
    • *
* * @param queueId the unique identifier of the source queue; must not be null or empty * @param consumer the message consumer that will process the message; must not be null - * @return {@code true} if a message was successfully consumed and processed, - * {@code false} if no message was available or processing failed - * @see MessageConsumer#accept(Object) - */ - default boolean consume(String queueId, MessageConsumer consumer) { - final Lease lease = receive(queueId); - if (lease == null) { - return false; - } - final boolean accepted = consumer.accept(lease.message()); - if (accepted) { - ack(queueId, lease.id()); - } - else { - release(queueId, lease.id()); - } - return accepted; - } - - /** - * A single delivered message paired with the token needed to renew, acknowledge - * or release it. The {@code id} is the queue-implementation specific handle - * (e.g. the Redis stream entry id) that identifies the delivered entry within - * its queue. - * - * @param the type of the delivered message - * @param id the implementation specific identifier of the delivered entry - * @param message the delivered message payload - */ - record Lease(String id, M message) {} - - /** - * Receives one message (either newly delivered or reclaimed from a stalled consumer) - * without acknowledging it. The caller becomes responsible for - * eventually calling {@link #ack(String, String)} once processing terminates, or - * {@link #release(String, String)} to hand it back for later redelivery. - * - * @param queueId the unique identifier of the source queue; must not be null or empty - * @return a {@link Lease} for the delivered message, or {@code null} if none is available - */ - Lease receive(String queueId); - - /** - * Resets the idle time of the given lease (heartbeat), so that an alive consumer - * keeps ownership of a message for as long as its handler runs. Implementations - * without a pending-entries list have no lease semantics and treat this as a no-op. - * - * @param queueId the unique identifier of the queue; must not be null or empty - * @param leaseId the identifier of the lease to renew + * @return the consumer's {@link MessageConsumer.Decision}, or {@code null} when no + * message was available + * @see MessageConsumer#accept(Object, MessageLease) */ - void renewLease(String queueId, String leaseId); - - /** - * Acknowledges terminal processing of the given lease, removing the message from - * the queue so that it is never redelivered. - * - * @param queueId the unique identifier of the queue; must not be null or empty - * @param leaseId the identifier of the lease to acknowledge - */ - void ack(String queueId, String leaseId); - - /** - * Releases the given lease without acknowledging it, so that the message becomes - * available for redelivery later (a nack; used on shutdown). Implementations - * without a pending-entries list re-offer the message. - * - * @param queueId the unique identifier of the queue; must not be null or empty - * @param leaseId the identifier of the lease to release - */ - void release(String queueId, String leaseId); - - /** - * How often an in-flight lease must be renewed to retain ownership, so an alive - * consumer is never reclaimed by a peer while its handler is still running. The - * value is the implementation's own setting (e.g. {@code visibility-timeout / 3} for a - * Redis consumer group) and MUST be shorter than the reclaim window. Returns - * {@code null} when the implementation has no lease concept (e.g. in-memory), in - * which case the caller uses its own default. - * - * @return the heartbeat interval, or {@code null} if the implementation has no lease - */ - default Duration heartbeatInterval() { - return null; - } - - /** - * Upper bound on a single {@code accept()} invocation before its lease is released - * (safety valve); it does not interrupt the handler thread. Returns {@code null} - * when the implementation has no lease concept, in which case the caller uses its - * own default. - * - * @return the maximum single-invocation processing time, or {@code null} - */ - default Duration maxProcessingTime() { - return null; - } + MessageConsumer.Decision consume(String queueId, MessageConsumer consumer); /** * Returns the approximate number of messages currently in the specified queue. @@ -234,6 +163,15 @@ default Duration maxProcessingTime() { * In a distributed environment with concurrent producers and consumers, the actual * number of messages may change immediately after this method returns.

* + *

Common use cases include:

+ *
    + *
  • Monitoring queue backlog and processing rates
    • + *
  • Capacity planning and resource allocation
    • + *
  • Alerting on queue growth beyond expected thresholds
    • + *
  • Load balancing decisions across consumer instances
    • + *
  • Testing and debugging queue behavior
    • + *
+ * *

Note: This operation may be expensive for large queues or distributed * implementations, so it should not be called excessively in performance-critical paths.

* diff --git a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/MicrometerQueueMetrics.java b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/MicrometerQueueMetrics.java index 0efc6ceb..9a38a9aa 100644 --- a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/MicrometerQueueMetrics.java +++ b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/MicrometerQueueMetrics.java @@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.IntSupplier; import java.util.function.ToDoubleFunction; @@ -40,21 +41,40 @@ * so that the library remains loadable on classpaths without Micrometer (consumers * that don't need metrics use {@link NoopQueueMetrics#INSTANCE}).

* - *

Published meters (all tagged with {@code queue=} and - * {@code queue_id=}): + *

Published meters (all tagged with {@code queue=}; per-queue meters + * additionally tagged {@code queue_id=}): *

    *
  • {@code seqera.workqueue.entries} (Gauge) — current backlog
    • *
  • {@code seqera.workqueue.messages} (Counter; tag {@code outcome=processed|active|errored})
    • *
  • {@code seqera.workqueue.processing} (Timer with percentile histogram; same outcome tag)
    • + *
  • {@code seqera.workqueue.leased} (Gauge) — entries currently leased (in-flight)
    • + *
  • {@code seqera.workqueue.lease.age.max} (Gauge) — age of the oldest leased entry, in seconds
    • + *
  • {@code seqera.workqueue.lease.renewal} (Timer) — lease-renewal tick duration
    • + *
  • {@code seqera.workqueue.lease.renewal.errors} (Counter) — failed renewal round-trips
    • + *
  • {@code seqera.workqueue.lease.renewal.age} (Gauge) — seconds since the last COMPLETED + * renewal tick; unbounded growth = a stuck renewal scheduler
    • + *
  • {@code seqera.workqueue.lease.lost} (Counter) — leases found owned by another consumer
    • + *
  • {@code seqera.workqueue.lease.leak} (Counter) — leases dropped by the age backstop
    • + *
  • {@code seqera.workqueue.saturated} (Counter) — polls skipped on a not-ready consumer
    • + *
  • {@code seqera.workqueue.deferred} (Counter) — deliveries deferred to a task-owned lease
    • *
*/ public final class MicrometerQueueMetrics implements QueueMetrics { private static final Logger log = LoggerFactory.getLogger(MicrometerQueueMetrics.class); - public static final String METRIC_BACKLOG = "seqera.workqueue.entries"; - public static final String METRIC_MESSAGES = "seqera.workqueue.messages"; - public static final String METRIC_PROCESSING = "seqera.workqueue.processing"; + public static final String METRIC_BACKLOG = "seqera.workqueue.entries"; + public static final String METRIC_MESSAGES = "seqera.workqueue.messages"; + public static final String METRIC_PROCESSING = "seqera.workqueue.processing"; + public static final String METRIC_LEASED = "seqera.workqueue.leased"; + public static final String METRIC_LEASE_AGE_MAX = "seqera.workqueue.lease.age.max"; + public static final String METRIC_RENEW_TICK = "seqera.workqueue.lease.renewal"; + public static final String METRIC_RENEW_ERRORS = "seqera.workqueue.lease.renewal.errors"; + public static final String METRIC_RENEW_AGE = "seqera.workqueue.lease.renewal.age"; + public static final String METRIC_LEASE_LOST = "seqera.workqueue.lease.lost"; + public static final String METRIC_LEASE_LEAK = "seqera.workqueue.lease.leak"; + public static final String METRIC_SATURATED = "seqera.workqueue.saturated"; + public static final String METRIC_DEFERRED = "seqera.workqueue.deferred"; private final MeterRegistry registry; private final String queueName; @@ -62,10 +82,38 @@ public final class MicrometerQueueMetrics implements QueueMetrics { // object through a WeakReference; without this map the supplier lambda would be // GC-eligible the moment bindBacklog returns and the gauge would report NaN. private final ConcurrentMap backlogSuppliers = new ConcurrentHashMap<>(); + // Backing values of the leased-entries and max-lease-age gauges, updated on every + // renewal tick. The fields themselves are the strong references that keep the gauge + // sources alive. + private final AtomicLong leasedEntries = new AtomicLong(); + private final AtomicLong maxLeaseAgeNanos = new AtomicLong(); public MicrometerQueueMetrics(MeterRegistry registry, String queueName) { this.registry = registry; this.queueName = queueName; + Gauge.builder(METRIC_LEASED, leasedEntries, AtomicLong::doubleValue) + .description("Number of queue entries currently leased (in-flight)") + .tag("queue", queueName) + .baseUnit("entries") + .register(registry); + Gauge.builder(METRIC_LEASE_AGE_MAX, maxLeaseAgeNanos, MicrometerQueueMetrics::nanosToSeconds) + .description("Age of the oldest currently-leased entry") + .tag("queue", queueName) + .baseUnit("seconds") + .register(registry); + } + + private static double nanosToSeconds(AtomicLong nanos) { + return nanos.get() / 1e9; + } + + @Override + public void bindRenewalLiveness(java.util.function.LongSupplier ageNanos) { + Gauge.builder(METRIC_RENEW_AGE, ageNanos, a -> a.getAsLong() / 1e9) + .description("Seconds since the last completed lease-renewal tick; unbounded growth means the renewal scheduler is stuck") + .tag("queue", queueName) + .baseUnit("seconds") + .register(registry); } @Override @@ -118,4 +166,55 @@ public void recordOutcome(long startNanos, String queueId, Outcome outcome) { .register(registry) .record(System.nanoTime() - startNanos, TimeUnit.NANOSECONDS); } + + @Override + public void renewTick(long durationNanos, int leasedCount, long maxAgeNanos) { + leasedEntries.set(leasedCount); + maxLeaseAgeNanos.set(maxAgeNanos); + Timer.builder(METRIC_RENEW_TICK) + .description("Duration of one lease-renewal tick") + .tag("queue", queueName) + .register(registry) + .record(durationNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void renewError() { + counter(METRIC_RENEW_ERRORS, "Failed lease-renewal round-trips").increment(); + } + + @Override + public void leaseLost() { + counter(METRIC_LEASE_LOST, "Leases found owned by another consumer during renewal").increment(); + } + + @Override + public void leaseLeak() { + counter(METRIC_LEASE_LEAK, "Leases dropped by the renewal age backstop").increment(); + } + + @Override + public void saturated(String queueId) { + queueCounter(METRIC_SATURATED, "Polls skipped because the consumer was not ready", queueId).increment(); + } + + @Override + public void deferred(String queueId) { + queueCounter(METRIC_DEFERRED, "Deliveries deferred to a task-owned lease", queueId).increment(); + } + + private Counter counter(String name, String description) { + return Counter.builder(name) + .description(description) + .tag("queue", queueName) + .register(registry); + } + + private Counter queueCounter(String name, String description, String queueId) { + return Counter.builder(name) + .description(description) + .tag("queue", queueName) + .tag("queue_id", queueId) + .register(registry); + } } diff --git a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/Outcome.java b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/Outcome.java index 78781f10..3cd226b3 100644 --- a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/Outcome.java +++ b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/Outcome.java @@ -22,13 +22,13 @@ * {@code seqera.workqueue.messages} counter and {@code seqera.workqueue.processing} timer. */ public enum Outcome { - /** Consumer.accept returned true; message was acknowledged and removed. */ + /** Consumer decided ACK; message was acknowledged and removed. */ PROCESSED("processed"), - /** Consumer.accept returned false; message remains available for redelivery. */ + /** Consumer decided RETRY or DEFERRED; message remains pending (leased or redeliverable). */ ACTIVE("active"), /** Exception escaped the consumer or the underlying queue implementation. */ ERRORED("errored"), - /** Receive found no message available. Not counted or timed. */ + /** Poll found no message available. Not counted or timed. */ EMPTY("empty"); private final String tag; diff --git a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/QueueMetrics.java b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/QueueMetrics.java index 322ab097..de24fb15 100644 --- a/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/QueueMetrics.java +++ b/lib-data-workqueue/src/main/java/io/seqera/data/workqueue/metrics/QueueMetrics.java @@ -18,11 +18,12 @@ package io.seqera.data.workqueue.metrics; import java.util.function.IntSupplier; +import java.util.function.LongSupplier; /** * Metrics handle consumed by {@code AbstractWorkQueue}. Deliberately neutral with * respect to Micrometer types so consumers without {@code micrometer-core} on the - * classpath can still load and instantiate work-queue subclasses. + * classpath can still load and instantiate queue subclasses. * *

Two implementations are provided: *

    @@ -62,7 +63,41 @@ public interface QueueMetrics { * {@link #recordOutcome(long, String, Outcome)} (nanoseconds, or 0 for no-op). */ long startSample(); - /** Record the outcome of one processing cycle. {@link Outcome#EMPTY} receives + /** Record the outcome of one processing cycle. {@link Outcome#EMPTY} polls * must not count toward the messages counter or contribute to the timer. */ void recordOutcome(long startNanos, String queueId, Outcome outcome); + + /** Record one lease-renewal tick: its duration, the number of entries currently + * leased and the age of the oldest lease (leased count and max age are published + * as gauges by instrumented implementations). */ + default void renewTick(long durationNanos, int leasedCount, long maxLeaseAgeNanos) { } + + /** Record a failed lease-renewal round-trip (retried on the next tick). */ + default void renewError() { } + + /** + * Bind a liveness probe for the lease-renewal scheduler: the age (nanos) of the + * last COMPLETED renewal tick. A stuck tick — e.g. a renewal thread blocked on an + * exhausted connection pool — shows as unbounded growth here while every other + * renewal signal stays silent (a blocked borrow never throws, so renewError never + * fires and the tick-overrun warn never runs). Alert on age above a few renewal + * periods. + */ + default void bindRenewalLiveness(LongSupplier ageNanos) { } + + /** Record a lease found to be owned by another consumer during the renewal + * ownership check — the residual duplicate-execution window, made observable. */ + default void leaseLost() { } + + /** Record a lease dropped by the renewal age backstop — a settlement path + * that never ran; the claim cycle recovers the entry. */ + default void leaseLeak() { } + + /** Record a poll skipped because the queue's consumer reported not + * {@code ready()} — an admission-blocked replica, distinct from an idle one. */ + default void saturated(String queueId) { } + + /** Record a delivery whose consumer returned {@code DEFERRED} — a task took + * the message lease and settles it later. */ + default void deferred(String queueId) { } } diff --git a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueDrainTest.groovy b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueDrainTest.groovy new file mode 100644 index 00000000..1f751233 --- /dev/null +++ b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueDrainTest.groovy @@ -0,0 +1,195 @@ +/* + * Copyright 2026, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.data.workqueue + +import java.time.Duration +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger + +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.random.LongRndKey +import jakarta.inject.Inject +import spock.lang.Specification +import static io.seqera.data.workqueue.MessageConsumer.Decision.ACK +/** + * Covers the cooperative shutdown contract: a consumer already running must be allowed to + * finish, because at that point it may be mid-way through work against resources the caller + * is about to tear down. + * + * @author Paolo Di Tommaso + */ +@MicronautTest(environments = ['test']) +class AbstractWorkQueueDrainTest extends Specification { + + @Inject + LocalWorkQueue target + + def 'awaitQuiescent should let an in-progress consumer finish without interrupting it'() { + given: + def id = "stream-${LongRndKey.rndHex()}" + def stream = new TestPlainQueue(target) + and: 'a consumer that is slow enough to still be running when the drain starts' + def entered = new CountDownLatch(1) + def completed = new AtomicBoolean(false) + def interrupted = new AtomicBoolean(false) + stream.addConsumer(id, { msg, lease -> + entered.countDown() + try { + Thread.sleep(500) + completed.set(true) + } + catch (InterruptedException e) { + interrupted.set(true) + Thread.currentThread().interrupt() + } + return ACK + }) + + when: 'a message is picked up and the drain begins while the consumer is still inside it' + stream.offer(id, 'one') + entered.await(5, TimeUnit.SECONDS) + def quiesced = stream.awaitQuiescent(Duration.ofSeconds(10)) + + then: 'the drain waits for it rather than cutting it short' + quiesced + completed.get() + !interrupted.get() + + cleanup: + stream.close() + } + + def 'awaitQuiescent should stop the dispatcher claiming further messages'() { + given: + def id = "stream-${LongRndKey.rndHex()}" + def stream = new TestPlainQueue(target) + def seen = new AtomicInteger() + def entered = new CountDownLatch(1) + stream.addConsumer(id, { msg, lease -> + seen.incrementAndGet() + entered.countDown() + Thread.sleep(300) + return ACK + }) + + when: 'two messages are queued but the drain starts during the first' + stream.offer(id, 'one') + entered.await(5, TimeUnit.SECONDS) + stream.offer(id, 'two') + stream.awaitQuiescent(Duration.ofSeconds(10)) + and: 'well past the poll interval, so a live dispatcher would have taken the second' + Thread.sleep(1_500) + + then: 'only the message already claimed was delivered' + seen.get() == 1 + + cleanup: + stream.close() + } + + def 'awaitQuiescent should report false when the consumer outlives the timeout'() { + given: + def id = "stream-${LongRndKey.rndHex()}" + def stream = new TestPlainQueue(target) + def entered = new CountDownLatch(1) + stream.addConsumer(id, { msg, lease -> + entered.countDown() + Thread.sleep(2_000) + return ACK + }) + + when: + stream.offer(id, 'one') + entered.await(5, TimeUnit.SECONDS) + def quiesced = stream.awaitQuiescent(Duration.ofMillis(200)) + + then: 'the caller is told the drain did not complete, and decides what to do next' + !quiesced + + cleanup: + stream.close() + } + + def 'close should drain cooperatively instead of interrupting the consumer'() { + given: 'this is the behaviour change - close() used to interrupt the dispatcher first' + def id = "stream-${LongRndKey.rndHex()}" + def stream = new TestPlainQueue(target) + def entered = new CountDownLatch(1) + def completed = new AtomicBoolean(false) + def interrupted = new AtomicBoolean(false) + stream.addConsumer(id, { msg, lease -> + entered.countDown() + try { + Thread.sleep(500) + completed.set(true) + } + catch (InterruptedException e) { + interrupted.set(true) + Thread.currentThread().interrupt() + } + return ACK + }) + + when: + stream.offer(id, 'one') + entered.await(5, TimeUnit.SECONDS) + stream.close() + + then: 'the consumer ran to completion and was never interrupted' + completed.get() + !interrupted.get() + } + + def 'awaitQuiescent should be a no-op when no consumer was ever registered'() { + given: + def stream = new TestPlainQueue(target) + + expect: + stream.awaitQuiescent(Duration.ofSeconds(1)) + + cleanup: + stream.close() + } + + def 'a second close should not wait again after the first one gave up'() { + given: 'a consumer that outlives the first close budget' + def id = "stream-${LongRndKey.rndHex()}" + def stream = new TestPlainQueue(target) + def entered = new CountDownLatch(1) + stream.addConsumer(id, { msg, lease -> + entered.countDown() + Thread.sleep(3_000) + return ACK + }) + + when: + stream.offer(id, 'one') + entered.await(5, TimeUnit.SECONDS) + stream.close(Duration.ofMillis(200)) + and: 'the @PreDestroy backstop closes again, with its own larger default budget' + def begin = System.currentTimeMillis() + stream.close() + def elapsed = System.currentTimeMillis() - begin + + then: 'the shutdown budget was spent once - the second close must not spend it again' + elapsed < 1_000 + } + +} diff --git a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueLocalTest.groovy b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueLocalTest.groovy index c57258ef..1bbd562e 100644 --- a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueLocalTest.groovy +++ b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueLocalTest.groovy @@ -20,10 +20,14 @@ package io.seqera.data.workqueue import io.seqera.random.LongRndKey import spock.lang.Specification +import java.time.Duration import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.atomic.AtomicInteger import io.micronaut.test.extensions.spock.annotation.MicronautTest import jakarta.inject.Inject +import static io.seqera.data.workqueue.MessageConsumer.Decision.ACK +import static io.seqera.data.workqueue.MessageConsumer.Decision.RETRY /** * * @author Paolo Di Tommaso @@ -36,23 +40,43 @@ class AbstractWorkQueueLocalTest extends Specification { def 'should offer and consume some messages' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" + def id1 = "stream-${LongRndKey.rndHex()}" and: - def queue = new TestQueue(target) - def sink = new ArrayBlockingQueue(10) + def stream = new TestQueue(target) + def queue = new ArrayBlockingQueue(10) and: - queue.addConsumer(id1, { it-> sink.add(it) }) + stream.addConsumer(id1, { it, lease -> queue.add(it); ACK }) when: - queue.offer(id1, new TestMessage('one','two')) - queue.offer(id1, new TestMessage('alpha','omega')) + stream.offer(id1, new TestMessage('one','two')) + stream.offer(id1, new TestMessage('alpha','omega')) then: - sink.take()==new TestMessage('one','two') - sink.take()==new TestMessage('alpha','omega') + queue.take()==new TestMessage('one','two') + queue.take()==new TestMessage('alpha','omega') + + cleanup: + stream.close() + } + + def 'a retrying consumer should be paced by the poll interval, not spin hot' () { + given: 'a local stream with ZERO retry delay, isolating the dispatcher pacing' + def id1 = "stream-${LongRndKey.rndHex()}" + def local = new LocalWorkQueue() + local.@retryDelay = Duration.ZERO + def stream = new TestQueue(local) // pollInterval = 1s + def invocations = new AtomicInteger() + + when: 'a single message whose consumer always asks for a retry' + stream.addConsumer(id1, { it, lease -> invocations.incrementAndGet(); RETRY }) + stream.offer(id1, new TestMessage('a', 'b')) + sleep 2_500 + + then: 'invocations are bounded by the poll cadence - a RETRY is not progress' + invocations.get() <= 4 cleanup: - queue.close() + stream.close() } } diff --git a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueMetricsTest.groovy b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueMetricsTest.groovy index b1d3013b..cd68a03f 100644 --- a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueMetricsTest.groovy +++ b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AbstractWorkQueueMetricsTest.groovy @@ -19,12 +19,17 @@ package io.seqera.data.workqueue import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference import io.micrometer.core.instrument.simple.SimpleMeterRegistry import io.seqera.random.LongRndKey import spock.lang.Specification import spock.util.concurrent.PollingConditions +import static io.seqera.data.workqueue.MessageConsumer.Decision.ACK +import static io.seqera.data.workqueue.MessageConsumer.Decision.DEFERRED +import static io.seqera.data.workqueue.MessageConsumer.Decision.RETRY /** * Verifies the Micrometer instrumentation in AbstractWorkQueue. @@ -37,19 +42,19 @@ import spock.util.concurrent.PollingConditions */ class AbstractWorkQueueMetricsTest extends Specification { - def 'should register backlog gauge tied to queue length'() { + def 'should register backlog gauge tied to stream length'() { given: def registry = new SimpleMeterRegistry() def target = new LocalWorkQueue() - def queue = TestQueue.withRegistry(target, registry) - def queueId = "queue-${LongRndKey.rndHex()}" - def sink = new LinkedBlockingQueue() + def stream = TestQueue.withRegistry(target, registry) + def queueId = "stream-${LongRndKey.rndHex()}" + def queue = new LinkedBlockingQueue() when: - queue.addConsumer(queueId, { msg -> sink.add(msg); true }) + stream.addConsumer(queueId, { msg, lease -> queue.add(msg); ACK }) // immediately offer two entries before the consumer thread drains them - queue.offer(queueId, new TestMessage('a','b')) - queue.offer(queueId, new TestMessage('c','d')) + stream.offer(queueId, new TestMessage('a','b')) + stream.offer(queueId, new TestMessage('c','d')) then: def gauge = registry.find('seqera.workqueue.entries') @@ -60,26 +65,26 @@ class AbstractWorkQueueMetricsTest extends Specification { // gauge value tracks the underlying length() — eventually 0 after drain new PollingConditions(timeout: 5).eventually { assert gauge.value() == 0d - assert sink.size() == 2 + assert queue.size() == 2 } cleanup: - queue.close() + stream.close() } def 'should increment processed counter and record timer on success'() { given: def registry = new SimpleMeterRegistry() def target = new LocalWorkQueue() - def queue = TestQueue.withRegistry(target, registry) - def queueId = "queue-${LongRndKey.rndHex()}" + def stream = TestQueue.withRegistry(target, registry) + def queueId = "stream-${LongRndKey.rndHex()}" def seen = new AtomicInteger() when: - queue.addConsumer(queueId, { msg -> seen.incrementAndGet(); true }) - queue.offer(queueId, new TestMessage('a','b')) - queue.offer(queueId, new TestMessage('c','d')) - queue.offer(queueId, new TestMessage('e','f')) + stream.addConsumer(queueId, { msg, lease -> seen.incrementAndGet(); ACK }) + stream.offer(queueId, new TestMessage('a','b')) + stream.offer(queueId, new TestMessage('c','d')) + stream.offer(queueId, new TestMessage('e','f')) then: new PollingConditions(timeout: 5).eventually { @@ -102,23 +107,23 @@ class AbstractWorkQueueMetricsTest extends Specification { } cleanup: - queue.close() + stream.close() } def 'should count consumer-rejected message as active'() { given: def registry = new SimpleMeterRegistry() def target = new LocalWorkQueue() - def queue = TestQueue.withRegistry(target, registry) - def queueId = "queue-${LongRndKey.rndHex()}" + def stream = TestQueue.withRegistry(target, registry) + def queueId = "stream-${LongRndKey.rndHex()}" def attempts = new AtomicInteger() when: - // first call returns false, then true — Local impl re-queues after the poll interval - queue.addConsumer(queueId, { msg -> - attempts.incrementAndGet() == 1 ? false : true + // first call retries, then acks — Local impl re-queues the message immediately + stream.addConsumer(queueId, { msg, lease -> + attempts.incrementAndGet() == 1 ? RETRY : ACK }) - queue.offer(queueId, new TestMessage('a','b')) + stream.offer(queueId, new TestMessage('a','b')) then: new PollingConditions(timeout: 8).eventually { @@ -137,26 +142,106 @@ class AbstractWorkQueueMetricsTest extends Specification { } cleanup: - queue.close() + stream.close() } def 'should register no meters when using the no-op 1-arg constructor'() { given: // 1-arg constructor → no metrics def target = new LocalWorkQueue() - def queue = new TestQueue(target) - def queueId = "queue-${LongRndKey.rndHex()}" - def sink = new LinkedBlockingQueue() + def stream = new TestQueue(target) + def queueId = "stream-${LongRndKey.rndHex()}" + def queue = new LinkedBlockingQueue() when: - queue.addConsumer(queueId, { msg -> sink.add(msg); true }) - queue.offer(queueId, new TestMessage('a','b')) + stream.addConsumer(queueId, { msg, lease -> queue.add(msg); ACK }) + stream.offer(queueId, new TestMessage('a','b')) then: // no exceptions, message still flows - sink.poll(5, TimeUnit.SECONDS) == new TestMessage('a','b') + queue.poll(5, TimeUnit.SECONDS) == new TestMessage('a','b') cleanup: - queue.close() + stream.close() + } + + def 'should skip a not-ready consumer and count the poll as saturated'() { + given: + def registry = new SimpleMeterRegistry() + def target = new LocalWorkQueue() + def stream = TestQueue.withRegistry(target, registry) + def queueId = "stream-${LongRndKey.rndHex()}" + def ready = new AtomicBoolean(false) + def queue = new LinkedBlockingQueue() + def consumer = new MessageConsumer() { + @Override + MessageConsumer.Decision accept(TestMessage msg, MessageLease lease) { + queue.add(msg) + return ACK + } + @Override + boolean ready() { + return ready.get() + } + } + + when: + stream.addConsumer(queueId, consumer) + stream.offer(queueId, new TestMessage('a','b')) + + then: 'the message is not claimed while the consumer is saturated' + queue.poll(1, TimeUnit.SECONDS) == null + target.length(queueId) == 1 + and: 'the skipped polls are counted as saturated, not empty' + new PollingConditions(timeout: 5).eventually { + def saturated = registry.find('seqera.workqueue.saturated') + .tag('queue', 'test-queue') + .tag('queue_id', queueId) + .counter() + assert saturated != null + assert saturated.count() >= 1.0d + } + + when: 'the admission gate opens' + ready.set(true) + + then: + queue.poll(5, TimeUnit.SECONDS) == new TestMessage('a','b') + + cleanup: + stream.close() + } + + def 'should count a deferred delivery as active plus a distinct deferred counter'() { + given: + def registry = new SimpleMeterRegistry() + def target = new LocalWorkQueue() + def stream = TestQueue.withRegistry(target, registry) + def queueId = "stream-${LongRndKey.rndHex()}" + def held = new AtomicReference() + + when: + stream.addConsumer(queueId, { msg, lease -> held.compareAndSet(null, lease); DEFERRED }) + stream.offer(queueId, new TestMessage('a','b')) + + then: + new PollingConditions(timeout: 5).eventually { + assert held.get() != null + + def deferred = registry.find('seqera.workqueue.deferred') + .tag('queue', 'test-queue') + .tag('queue_id', queueId) + .counter() + def active = registry.find('seqera.workqueue.messages') + .tag('outcome', 'active') + .tag('queue_id', queueId) + .counter() + assert deferred?.count() >= 1.0d + assert active?.count() >= 1.0d + } + + cleanup: + held.get()?.ack() + stream.close() } } diff --git a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AsyncWorkQueueLocalTest.groovy b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AsyncWorkQueueLocalTest.groovy deleted file mode 100644 index 8884c125..00000000 --- a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/AsyncWorkQueueLocalTest.groovy +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Copyright 2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.data.workqueue - -import java.time.Duration -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -import io.seqera.random.LongRndKey -import spock.lang.Specification -import spock.util.concurrent.PollingConditions - -/** - * Async-processing behaviour of {@link AbstractWorkQueue} exercised over the - * in-memory {@link LocalWorkQueue} backend, so these run WITHOUT Docker. - * - * Covers: non-blocking dispatch, concurrency, re-poll cadence, serial-per-command, - * backpressure, and concurrency==1 default. - * - * @author Paolo Di Tommaso - */ -class AsyncWorkQueueLocalTest extends Specification { - - // a slow handler on queue A must not delay a fast handler on queue B - def 'should not block a fast queue behind a slow one' () { - given: - def target = new LocalWorkQueue() - def queue = new TunableQueue(target, concurrency: 2, pollInterval: Duration.ofMillis(100)) - def idA = "queue-${LongRndKey.rndHex()}" - def idB = "queue-${LongRndKey.rndHex()}" - def slowDone = new CountDownLatch(1) - def fastDone = new CountDownLatch(1) - - when: - queue.addConsumer(idA, { msg -> Thread.sleep(3_000); slowDone.countDown(); true }) - queue.addConsumer(idB, { msg -> fastDone.countDown(); true }) - and: - queue.offer(idA, 'slow') - queue.offer(idB, 'fast') - - then: - // the fast handler completes well before the slow one finishes - fastDone.await(2, TimeUnit.SECONDS) - slowDone.count == 1 - - cleanup: - queue.close() - } - - // N messages with a slow handler complete in ~max(handler), not ~sum - def 'should process messages concurrently' () { - given: - def target = new LocalWorkQueue() - def queue = new TunableQueue(target, concurrency: 4, pollInterval: Duration.ofMillis(100)) - def id = "queue-${LongRndKey.rndHex()}" - def done = new CountDownLatch(4) - - when: - queue.addConsumer(id, { msg -> Thread.sleep(500); done.countDown(); true }) - def t0 = System.currentTimeMillis() - 4.times { queue.offer(id, "msg-$it".toString()) } - - then: - done.await(5, TimeUnit.SECONDS) - def elapsed = System.currentTimeMillis() - t0 - // 4 x 500ms serial would be ~2000ms; concurrent should be well under that - elapsed < 1_500 - - cleanup: - queue.close() - } - - // a not-yet-terminal command is re-invoked at ~pollInterval (Model B) - def 'should re-poll a not-yet-terminal command at poll interval' () { - given: - def poll = Duration.ofMillis(300) - def target = new LocalWorkQueue() - def queue = new TunableQueue(target, concurrency: 1, pollInterval: poll) - def id = "queue-${LongRndKey.rndHex()}" - def timestamps = new ConcurrentLinkedQueue() - - when: - // record the wall-clock of each invocation; stay non-terminal for 5 calls, then ack - queue.addConsumer(id, { msg -> - timestamps.add(System.currentTimeMillis()) - return timestamps.size() >= 5 - }) - queue.offer(id, 'running') - - then: - new PollingConditions(timeout: 10).eventually { - assert timestamps.size() == 5 - } - and: - def times = timestamps.toList() - def gaps = (1..= 150 && it <= 1_500 } - - cleanup: - queue.close() - } - - // never two concurrent accept() invocations for the same command - def 'should invoke a command strictly serially across re-polls' () { - given: - def target = new LocalWorkQueue() - def queue = new TunableQueue(target, concurrency: 4, pollInterval: Duration.ofMillis(150)) - def id = "queue-${LongRndKey.rndHex()}" - def inProgress = new AtomicInteger() - def maxConcurrent = new AtomicInteger() - def calls = new AtomicInteger() - - when: - queue.addConsumer(id, { msg -> - def now = inProgress.incrementAndGet() - maxConcurrent.accumulateAndGet(now, Math::max) - Thread.sleep(100) - inProgress.decrementAndGet() - return calls.incrementAndGet() >= 4 - }) - queue.offer(id, 'running') - - then: - new PollingConditions(timeout: 10).eventually { - assert calls.get() >= 4 - } - and: - // one message => the same lease is never processed by two workers at once - maxConcurrent.get() == 1 - - cleanup: - queue.close() - } - - // with pool size K and more than K ready messages, at most K run at once - def 'should bound concurrent handlers by the pool size (backpressure)' () { - given: - def target = new LocalWorkQueue() - def queue = new TunableQueue(target, concurrency: 2, pollInterval: Duration.ofMillis(100)) - def id = "queue-${LongRndKey.rndHex()}" - def inProgress = new AtomicInteger() - def maxConcurrent = new AtomicInteger() - def done = new CountDownLatch(6) - - when: - queue.addConsumer(id, { msg -> - def now = inProgress.incrementAndGet() - maxConcurrent.accumulateAndGet(now, Math::max) - Thread.sleep(300) - inProgress.decrementAndGet() - done.countDown() - true - }) - 6.times { queue.offer(id, "msg-$it".toString()) } - - then: - done.await(10, TimeUnit.SECONDS) - maxConcurrent.get() <= 2 - - cleanup: - queue.close() - } - - // default concurrency is 1: at most one handler runs at a time - def 'should run at most one handler with the default concurrency' () { - given: - def target = new LocalWorkQueue() - // default TunableQueue -> concurrency 1 - def queue = new TunableQueue(target, pollInterval: Duration.ofMillis(100)) - def id = "queue-${LongRndKey.rndHex()}" - def inProgress = new AtomicInteger() - def maxConcurrent = new AtomicInteger() - def done = new CountDownLatch(4) - - when: - queue.addConsumer(id, { msg -> - def now = inProgress.incrementAndGet() - maxConcurrent.accumulateAndGet(now, Math::max) - Thread.sleep(150) - inProgress.decrementAndGet() - done.countDown() - true - }) - 4.times { queue.offer(id, "msg-$it".toString()) } - - then: - done.await(10, TimeUnit.SECONDS) - maxConcurrent.get() == 1 - - cleanup: - queue.close() - } - - // self-reclaim: if the heartbeat falls behind, this instance's own receive() (XAUTOCLAIM) - // can re-deliver an entry it is still processing. That duplicate must NOT start a second - // handler or leak a permit (regression for the concurrency()>1 permit-leak / double-run). - def 'self-reclaim of an in-flight entry does not double-run the handler'() { - given: 'a backing queue that re-delivers the SAME lease id twice, then nothing' - def deliveries = new AtomicInteger(0) - def acks = new AtomicInteger(0) - def target = [ - init : { String q -> }, - offer : { String q, String m -> }, - receive : { String q -> deliveries.getAndIncrement() < 2 ? new WorkQueue.Lease('dup-id', 'payload') : null }, - renewLease: { String q, String id -> }, - ack : { String q, String id -> acks.incrementAndGet() }, - release : { String q, String id -> }, - length : { String q -> 0 } - ] as WorkQueue - // concurrency 2 so the dispatcher can poll again while the first handler is in flight - def queue = new TunableQueue(target, concurrency: 2, pollInterval: Duration.ofMillis(50)) - def runs = new AtomicInteger(0) - def gate = new CountDownLatch(1) - - when: 'the handler blocks, so the entry stays in flight across the duplicate delivery' - queue.addConsumer('q1', { msg -> runs.incrementAndGet(); gate.await(5, TimeUnit.SECONDS); true } as MessageConsumer) - and: 'wait until the duplicate delivery has been attempted, then let a stray 2nd run surface' - new PollingConditions(timeout: 3).eventually { deliveries.get() >= 2 } - sleep(300) - - then: 'the handler ran exactly once despite the duplicate delivery' - runs.get() == 1 - - when: 'the handler completes' - gate.countDown() - - then: 'the entry is acked exactly once and the queue keeps functioning (no permit leak)' - new PollingConditions(timeout: 5).eventually { acks.get() == 1 } - - cleanup: - gate.countDown() - queue.close() - } - -} diff --git a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/LocalWorkQueueTest.groovy b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/LocalWorkQueueTest.groovy index ac0f61fb..52ba61a5 100644 --- a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/LocalWorkQueueTest.groovy +++ b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/LocalWorkQueueTest.groovy @@ -17,165 +17,234 @@ package io.seqera.data.workqueue +import java.time.Duration + import io.seqera.random.LongRndKey import spock.lang.Specification +import io.micronaut.context.annotation.Property import io.micronaut.test.extensions.spock.annotation.MicronautTest +import jakarta.inject.Inject +import static io.seqera.data.workqueue.MessageConsumer.Decision.ACK +import static io.seqera.data.workqueue.MessageConsumer.Decision.DEFERRED +import static io.seqera.data.workqueue.MessageConsumer.Decision.RETRY /** * * @author Paolo Di Tommaso */ @MicronautTest(environments = ['test']) +@Property(name = 'workqueue.local.retry-delay', value = '250ms') class LocalWorkQueueTest extends Specification { + @Inject + LocalWorkQueue contextQueue + + def 'the retry delay should bind from configuration' () { + expect: 'the @Value binding resolved the property - a key typo would silently fall back to 1s' + contextQueue.@retryDelay == Duration.ofMillis(250) + } + def 'should offer and consume a value' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" - def id2 = "queue-${LongRndKey.rndHex()}" + def id1 = "stream-${LongRndKey.rndHex()}" + def id2 = "stream-${LongRndKey.rndHex()}" and: - def queue = new LocalWorkQueue() + def stream = new LocalWorkQueue() and: - queue.init(id1) - queue.init(id2) + stream.init(id1) + stream.init(id2) when: - queue.offer(id1, 'one') + stream.offer(id1, 'one') and: - queue.offer(id2, 'alpha') - queue.offer(id2, 'delta') - queue.offer(id2, 'gamma') + stream.offer(id2, 'alpha') + stream.offer(id2, 'delta') + stream.offer(id2, 'gamma') then: - queue.consume(id1, { it-> it=='one'}) + stream.consume(id1, { it, lease -> assert it=='one'; ACK }) == ACK and: - queue.consume(id2, { it-> it=='alpha'}) - queue.consume(id2, { it-> it=='delta'}) - queue.consume(id2, { it-> it=='gamma'}) + stream.consume(id2, { it, lease -> assert it=='alpha'; ACK }) == ACK + stream.consume(id2, { it, lease -> assert it=='delta'; ACK }) == ACK + stream.consume(id2, { it, lease -> assert it=='gamma'; ACK }) == ACK and: - !queue.consume(id2, { it-> assert false /* <-- this should not be invoked */ }) + stream.consume(id2, { it, lease -> assert false /* <-- this should not be invoked */ }) == null } def 'should offer and consume a value with a failure' () { - given: - def id1 = "queue-${LongRndKey.rndHex()}" - def queue = new LocalWorkQueue() - queue.init(id1) + given: 'a zero retry delay: this test covers settlement ordering, not pacing' + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalWorkQueue() + stream.@retryDelay = Duration.ZERO + stream.init(id1) when: - queue.offer(id1, 'alpha') - queue.offer(id1, 'delta') - queue.offer(id1, 'gamma') + stream.offer(id1, 'alpha') + stream.offer(id1, 'delta') + stream.offer(id1, 'gamma') then: - queue.consume(id1, { it-> it=='alpha'}) + stream.consume(id1, { it, lease -> assert it=='alpha'; ACK }) == ACK + and: + // a consumer throw settles as RETRY - the message is re-queued at the tail + stream.consume(id1, { it, lease -> throw new RuntimeException("Oops") }) == RETRY and: - // the default consume() does not catch handler exceptions: it propagates and, - // since receive() already removed 'delta' and release() is not reached, it is dropped - try { - queue.consume(id1, { it-> throw new RuntimeException("Oops")}) - assert false - } - catch (RuntimeException e) { - assert e.message == 'Oops' - } + // next message is 'gamma' as expected + stream.consume(id1, { it, lease -> assert it=='gamma'; ACK }) == ACK and: - // next message is 'gamma' as expected ('delta' was dropped on the throw) - queue.consume(id1, { it-> it=='gamma'}) + // now the errored message is available again + stream.consume(id1, { it, lease -> assert it=='delta'; ACK }) == ACK and: - !queue.consume(id1, { it-> assert false /* <-- this should not be invoked */ }) + stream.consume(id1, { it, lease -> assert false /* <-- this should not be invoked */ }) == null when: - queue.offer(id1, 'something') + stream.offer(id1, 'something') then: - queue.consume(id1, { it-> it=='something'}) + stream.consume(id1, { it, lease -> assert it=='something'; ACK }) == ACK } def 'should validate length method' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" - def queue = new LocalWorkQueue() - queue.init(id1) + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalWorkQueue() + stream.init(id1) expect: - queue.length(id1) == 0 + stream.length(id1) == 0 when: - queue.offer(id1, 'alpha') - queue.offer(id1, 'delta') - queue.offer(id1, 'gamma') + stream.offer(id1, 'alpha') + stream.offer(id1, 'delta') + stream.offer(id1, 'gamma') then: - queue.length(id1) == 3 + stream.length(id1) == 3 when: - queue.consume(id1, { it-> true}) + stream.consume(id1, { it, lease -> ACK }) then: - queue.length(id1) == 2 + stream.length(id1) == 2 } - // Local backend: receive returns a lease; renewLease is a no-op; release re-offers - def 'should receive and release re-offering the message' () { - given: - def id1 = "queue-${LongRndKey.rndHex()}" - def queue = new LocalWorkQueue() - queue.init(id1) - queue.offer(id1, 'alpha') - - when: 'receive takes the message off the queue (lease id == message value)' - def lease = queue.receive(id1) - then: - lease != null - lease.message() == 'alpha' - queue.length(id1) == 0 + def 'deferred message should stay unavailable until the lease settles' () { + given: 'a zero retry delay: this test covers settlement semantics, not pacing' + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalWorkQueue() + stream.@retryDelay = Duration.ZERO + stream.init(id1) + MessageLease held = null - when: 'renewLease is a no-op and does not throw nor alter the queue' - queue.renewLease(id1, lease.id()) + when: + stream.offer(id1, 'alpha') then: - queue.length(id1) == 0 + stream.consume(id1, { it, lease -> held = lease; DEFERRED }) == DEFERRED + and: 'the message is neither queued nor redeliverable while the lease is open' + stream.length(id1) == 0 + stream.consume(id1, { it, lease -> assert false /* <-- this should not be invoked */ }) == null - when: 'release re-offers the message for later redelivery' - queue.release(id1, lease.id()) + when: 'retry makes it redeliverable (zero delay here)' + held.retry() then: - queue.length(id1) == 1 - queue.receive(id1).message() == 'alpha' + stream.length(id1) == 1 + stream.consume(id1, { it, lease -> assert it=='alpha'; ACK }) == ACK } - def 'should ack by dropping the received message' () { + def 'ack should remove a deferred message' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" - def queue = new LocalWorkQueue() - queue.init(id1) - queue.offer(id1, 'alpha') + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalWorkQueue() + stream.init(id1) + MessageLease held = null when: - def lease = queue.receive(id1) - queue.ack(id1, lease.id()) + stream.offer(id1, 'alpha') + stream.consume(id1, { it, lease -> held = lease; DEFERRED }) + and: 'settle from a different thread' + def settler = new Thread({ held.ack() }) + settler.start() + settler.join() then: - // ack is a no-op (already removed on receive) and nothing is redelivered - queue.length(id1) == 0 - queue.receive(id1) == null + stream.length(id1) == 0 + stream.consume(id1, { it, lease -> assert false /* <-- this should not be invoked */ }) == null } - // default consume() acks on true (message gone) / releases on false (redelivered) - def 'should ack on true and release on false via default consume()' () { + def 'double settlement should be a no-op' () { given: - def id1 = "queue-${LongRndKey.rndHex()}" - def queue = new LocalWorkQueue() - queue.init(id1) - queue.offer(id1, 'keep-me') + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalWorkQueue() + stream.init(id1) + MessageLease held = null - when: 'consumer returns false -> message is released and stays available' - def r1 = queue.consume(id1, { it -> false }) + when: + stream.offer(id1, 'alpha') + stream.consume(id1, { it, lease -> held = lease; DEFERRED }) + and: 'first call wins - the late retry cannot resurrect the acked message' + held.ack() + held.retry() + held.retry() then: - !r1 - queue.length(id1) == 1 + stream.length(id1) == 0 + stream.consume(id1, { it, lease -> assert false /* <-- this should not be invoked */ }) == null + } - when: 'consumer returns true -> message is acked and removed' - def r2 = queue.consume(id1, { it -> it == 'keep-me' }) + def 'retry should delay redelivery by the local retry delay' () { + given: 'a stream with a 300ms retry delay - the local analog of the Redis claim cadence' + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalWorkQueue() + stream.@retryDelay = Duration.ofMillis(300) + stream.init(id1) + MessageLease held = null + + when: + stream.offer(id1, 'alpha') + stream.consume(id1, { it, lease -> held = lease; DEFERRED }) + held.retry() + then: 'the message is NOT redeliverable before the delay - no hot retry loop' + stream.consume(id1, { it, lease -> assert false /* <-- this should not be invoked */ }) == null + and: 'it becomes redeliverable once the delay elapses' + sleep 400 + stream.consume(id1, { it, lease -> assert it=='alpha'; ACK }) == ACK + } + + def 'retryAfter should delay redelivery by the requested delay' () { + given: 'a stream whose plain retry delay is tiny, so the explicit delay is what gates' + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalWorkQueue() + stream.@retryDelay = Duration.ofMillis(10) + stream.init(id1) + MessageLease held = null + + when: + stream.offer(id1, 'alpha') + stream.consume(id1, { it, lease -> held = lease; DEFERRED }) + held.retryAfter(Duration.ofMillis(400)) + then: 'not redeliverable before the requested delay' + sleep 100 + stream.consume(id1, { it, lease -> assert false /* <-- this should not be invoked */ }) == null + and: 'redeliverable once it elapses' + sleep 400 + stream.consume(id1, { it, lease -> assert it=='alpha'; ACK }) == ACK + and: 'a late double-settlement is a no-op' + held.retry() + stream.length(id1) == 0 + } + + def 'consumer throw should settle as retry leaving nothing leased' () { + given: 'a zero retry delay: this test covers settlement semantics, not pacing' + def id1 = "stream-${LongRndKey.rndHex()}" + def stream = new LocalWorkQueue() + stream.@retryDelay = Duration.ZERO + stream.init(id1) + MessageLease held = null + + when: + stream.offer(id1, 'alpha') then: - r2 - queue.length(id1) == 0 - and: - // nothing left to consume - !queue.consume(id1, { it -> assert false /* not invoked */ }) + stream.consume(id1, { it, lease -> held = lease; throw new RuntimeException('Oops') }) == RETRY + and: 'the message is redeliverable (zero delay here)' + stream.length(id1) == 1 + and: 'the lease is already settled - a late retry cannot duplicate the message' + held.retry() + stream.length(id1) == 1 + stream.consume(id1, { it, lease -> assert it=='alpha'; ACK }) == ACK } } diff --git a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/QueueMetricsClassloaderTest.groovy b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/QueueMetricsClassloaderTest.groovy index 524ba383..6b68343c 100644 --- a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/QueueMetricsClassloaderTest.groovy +++ b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/QueueMetricsClassloaderTest.groovy @@ -74,10 +74,10 @@ class QueueMetricsClassloaderTest extends Specification { def isolated = new URLClassLoader(classpathWithoutMicrometer(), ClassLoader.platformClassLoader) when: 'classes load through the isolated loader' - Class localCls = Class.forName('io.seqera.data.workqueue.LocalWorkQueue', true, isolated) - Class workQueueIfc = Class.forName('io.seqera.data.workqueue.WorkQueue', true, isolated) - Class subclassCls = Class.forName('io.seqera.data.workqueue.TestPlainQueue', true, isolated) - Class abstractCls = Class.forName('io.seqera.data.workqueue.AbstractWorkQueue', true, isolated) + Class localCls = Class.forName('io.seqera.data.workqueue.LocalWorkQueue', true, isolated) + Class workQueueIfc = Class.forName('io.seqera.data.workqueue.WorkQueue', true, isolated) + Class subclassCls = Class.forName('io.seqera.data.workqueue.TestPlainQueue', true, isolated) + Class abstractCls = Class.forName('io.seqera.data.workqueue.AbstractWorkQueue', true, isolated) then: 'every class came from the isolated loader, not the parent' localCls.classLoader.is(isolated) diff --git a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/TestQueue.groovy b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/TestQueue.groovy index fa5f61fa..ae951318 100644 --- a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/TestQueue.groovy +++ b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/TestQueue.groovy @@ -34,12 +34,10 @@ class TestQueue extends AbstractWorkQueue { TestQueue(WorkQueue target) { super(target) - withHandlerExecutor(TestWorkerPool.INSTANCE) } TestQueue(WorkQueue target, QueueMetrics metrics) { super(target, metrics) - withHandlerExecutor(TestWorkerPool.INSTANCE) } static TestQueue withRegistry(WorkQueue target, MeterRegistry registry) { @@ -53,7 +51,7 @@ class TestQueue extends AbstractWorkQueue { String encode(TestMessage message) { return new JsonBuilder([x: message.x, y: message.y]).toString() } - + @Override TestMessage decode(String encoded) { def json = new JsonSlurper().parseText(encoded) diff --git a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/TunableQueue.groovy b/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/TunableQueue.groovy deleted file mode 100644 index 4e2523a9..00000000 --- a/lib-data-workqueue/src/test/groovy/io/seqera/data/workqueue/TunableQueue.groovy +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.data.workqueue - -import java.time.Duration - -import io.seqera.serde.encode.StringEncodingStrategy - -/** - * A {@link AbstractWorkQueue} used by the async-processing tests. It carries a - * String payload (identity encoding) and exposes the async knobs — concurrency, - * poll interval, heartbeat interval and max-processing-time — as constructor options - * so each test can tune them independently. - * - * @author Paolo Di Tommaso - */ -class TunableQueue extends AbstractWorkQueue { - - private final int workers - private final Duration pollDelay - private final Duration hbInterval - private final Duration maxProcTime - - TunableQueue(Map opts = [:], WorkQueue target) { - super(target) - withHandlerExecutor(TestWorkerPool.INSTANCE) - this.workers = (opts.concurrency ?: 1) as int - this.pollDelay = (opts.pollInterval ?: Duration.ofSeconds(1)) as Duration - this.hbInterval = (opts.heartbeatInterval ?: Duration.ofSeconds(20)) as Duration - this.maxProcTime = (opts.maxProcessingTime ?: Duration.ofMinutes(15)) as Duration - } - - @Override - protected StringEncodingStrategy createEncodingStrategy() { - return new StringEncodingStrategy() { - @Override - String encode(String message) { return message } - @Override - String decode(String encoded) { return encoded } - } - } - - @Override - protected String name() { - return 'tunable-queue' - } - - @Override - protected Duration pollInterval() { - return pollDelay - } - - @Override - protected int concurrency() { - return workers - } - - @Override - protected Duration heartbeatInterval() { - return hbInterval - } - - @Override - protected Duration maxProcessingTime() { - return maxProcTime - } -} diff --git a/lib-data-workqueue/src/test/java/io/seqera/data/workqueue/TestPlainQueue.java b/lib-data-workqueue/src/test/java/io/seqera/data/workqueue/TestPlainQueue.java index cb96ec3f..5ddf5afa 100644 --- a/lib-data-workqueue/src/test/java/io/seqera/data/workqueue/TestPlainQueue.java +++ b/lib-data-workqueue/src/test/java/io/seqera/data/workqueue/TestPlainQueue.java @@ -33,7 +33,6 @@ public class TestPlainQueue extends AbstractWorkQueue { public TestPlainQueue(WorkQueue target) { super(target); - withHandlerExecutor(TestWorkerPool.INSTANCE); } @Override diff --git a/lib-data-workqueue/src/test/java/io/seqera/data/workqueue/TestWorkerPool.java b/lib-data-workqueue/src/test/java/io/seqera/data/workqueue/TestWorkerPool.java deleted file mode 100644 index af5bd1ef..00000000 --- a/lib-data-workqueue/src/test/java/io/seqera/data/workqueue/TestWorkerPool.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2026, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.seqera.data.workqueue; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * Shared daemon handler executor for tests. {@link AbstractWorkQueue} no longer ships a - * built-in default executor (handlers must be supplied via {@code withHandlerExecutor}), so the - * test fixtures inject this one. Daemon threads so it never keeps the test JVM alive. - */ -public final class TestWorkerPool { - private TestWorkerPool() {} - - public static final ExecutorService INSTANCE = Executors.newCachedThreadPool(r -> { - Thread t = new Thread(r, "test-handler"); - t.setDaemon(true); - return t; - }); -}