From 0268578a3f51a410476b452c1548befe2be9f3ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Kobyli=C5=84ski?= Date: Sat, 26 Sep 2026 11:38:27 +0200 Subject: [PATCH 1/5] fix: validate deliverer configuration at startup Reject duplicate MessageDeliverer types instead of silently replacing one transport. Fail default processor startup when no deliverer is registered while preserving publisher-only and custom processor configurations. Closes #117 Closes #118 --- .../okapi/core/CompositeMessageDeliverer.kt | 14 +++- .../core/CompositeMessageDelivererTest.kt | 14 ++++ .../springboot/OutboxAutoConfiguration.kt | 12 +++ .../OutboxProcessorAutoConfigurationTest.kt | 82 ++++++++++++++++++- 4 files changed, 120 insertions(+), 2 deletions(-) diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/CompositeMessageDeliverer.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/CompositeMessageDeliverer.kt index 5084b98..d3bbc0e 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/CompositeMessageDeliverer.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/CompositeMessageDeliverer.kt @@ -19,7 +19,19 @@ class CompositeMessageDeliverer @JvmOverloads constructor( ) : MessageDeliverer { override val type: String = "composite" - private val registry: Map = deliverers.associateBy { it.type } + private val registry: Map = buildMap { + deliverers.forEach { deliverer -> + val type = deliverer.type + val previous = this[type] + if (previous != null) { + throw IllegalArgumentException( + "Duplicate MessageDeliverer type '$type': " + + "${previous.javaClass.name} and ${deliverer.javaClass.name}", + ) + } + put(type, deliverer) + } + } override fun deliver(entry: OutboxEntry): DeliveryResult { val messageDeliverer = registry[entry.deliveryType] diff --git a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/CompositeMessageDelivererTest.kt b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/CompositeMessageDelivererTest.kt index 43ac96d..0cd33db 100644 --- a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/CompositeMessageDelivererTest.kt +++ b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/CompositeMessageDelivererTest.kt @@ -1,5 +1,6 @@ package com.softwaremill.okapi.core +import io.kotest.assertions.throwables.shouldThrow import io.kotest.assertions.withClue import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe @@ -45,6 +46,19 @@ private fun rendezvousDeliverer(t: String, barrier: CyclicBarrier) = batchDelive } class CompositeMessageDelivererTest : FunSpec({ + test("duplicate deliverer types fail at construction") { + val error = shouldThrow { + CompositeMessageDeliverer( + listOf( + fixedDeliverer("kafka", DeliveryResult.Success), + fixedDeliverer("kafka", DeliveryResult.Success), + ), + ) + } + + error.message shouldContain "Duplicate MessageDeliverer type 'kafka'" + } + test("deliverBatch groups entries by type, delegates to each transport, preserves input order") { val composite = CompositeMessageDeliverer( listOf( diff --git a/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt b/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt index 8d0556b..eb6be69 100644 --- a/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt +++ b/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt @@ -18,6 +18,8 @@ import org.springframework.beans.factory.BeanFactory import org.springframework.beans.factory.BeanNotOfRequiredTypeException import org.springframework.beans.factory.NoSuchBeanDefinitionException import org.springframework.beans.factory.ObjectProvider +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Value import org.springframework.beans.factory.getBean import org.springframework.boot.autoconfigure.AutoConfiguration import org.springframework.boot.autoconfigure.condition.ConditionalOnClass @@ -79,6 +81,12 @@ class OutboxAutoConfiguration( private val primaryDataSource: DataSource, private val okapiProperties: OkapiProperties, ) { + @field:Value("\${okapi.processor.enabled:true}") + private var processorEnabled: Boolean = true + + @field:Autowired + private lateinit var messageDeliverers: ObjectProvider + private fun resolveDataSource(): DataSource = resolveDataSource(dataSources, primaryDataSource, okapiProperties) @Bean @@ -185,6 +193,10 @@ class OutboxAutoConfiguration( listener: ObjectProvider, clock: ObjectProvider, ): OutboxProcessor { + check(!processorEnabled || messageDeliverers.iterator().hasNext()) { + "Okapi processor is enabled but no MessageDeliverer bean is registered. " + + "Define a MessageDeliverer or set okapi.processor.enabled=false for publisher-only use." + } return OutboxProcessor( store = outboxStore, entryProcessor = outboxEntryProcessor, diff --git a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt index 25d9cc6..726ce57 100644 --- a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt +++ b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt @@ -6,7 +6,10 @@ import com.softwaremill.okapi.core.MessageDeliverer import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxEntryProcessor import com.softwaremill.okapi.core.OutboxMessage +import com.softwaremill.okapi.core.OutboxProcessor +import com.softwaremill.okapi.core.OutboxPublisher import com.softwaremill.okapi.core.OutboxStore +import com.softwaremill.okapi.core.RetryPolicy import com.softwaremill.okapi.core.TransactionRunner import com.softwaremill.okapi.core.TransportDispatch import com.softwaremill.okapi.micrometer.MicrometerOutboxListener @@ -18,11 +21,15 @@ import io.kotest.matchers.collections.shouldContain import io.kotest.matchers.collections.shouldNotBeEmpty import io.kotest.matchers.nulls.shouldNotBeNull import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain import org.springframework.beans.factory.getBean import org.springframework.boot.autoconfigure.AutoConfigurations import org.springframework.boot.autoconfigure.AutoConfigureAfter import org.springframework.boot.test.context.runner.ApplicationContextRunner +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration import org.springframework.jdbc.datasource.SimpleDriverDataSource +import java.time.Clock import java.time.Duration import java.time.Duration.ofMillis import java.time.Duration.ofMinutes @@ -54,6 +61,60 @@ class OutboxProcessorAutoConfigurationTest : FunSpec({ } } + test("enabled processor fails startup without a deliverer") { + ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(OutboxAutoConfiguration::class.java)) + .withBean(OutboxStore::class.java, { stubStore() }) + .withBean(DataSource::class.java, { SimpleDriverDataSource() }) + .withBean(TransactionRunner::class.java, { noOpTransactionRunner() }) + .run { ctx -> + generateSequence(ctx.startupFailure.shouldNotBeNull()) { it.cause } + .mapNotNull { it.message } + .joinToString(" ") shouldContain "no MessageDeliverer bean is registered" + } + } + + test("publisher-only application starts without a deliverer") { + ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(OutboxAutoConfiguration::class.java)) + .withBean(OutboxStore::class.java, { stubStore() }) + .withBean(DataSource::class.java, { SimpleDriverDataSource() }) + .withPropertyValues("okapi.processor.enabled=false", "okapi.purger.enabled=false") + .run { ctx -> + ctx.startupFailure shouldBe null + ctx.getBean(OutboxPublisher::class.java).shouldNotBeNull() + ctx.containsBean("outboxProcessorScheduler") shouldBe false + } + } + + test("custom processor starts without a MessageDeliverer bean") { + ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(OutboxAutoConfiguration::class.java)) + .withBean(OutboxStore::class.java, { stubStore() }) + .withBean(DataSource::class.java, { SimpleDriverDataSource() }) + .withBean(TransactionRunner::class.java, { noOpTransactionRunner() }) + .withBean(OutboxProcessor::class.java, { + OutboxProcessor( + stubStore(), + OutboxEntryProcessor(stubDeliverer(), RetryPolicy(maxRetries = 0), Clock.systemUTC()), + ) + }) + .run { ctx -> + ctx.startupFailure shouldBe null + ctx.getBean(OutboxProcessorScheduler::class.java).shouldNotBeNull() + } + } + + test("custom processor can use the auto-configured entry processor") { + contextRunner + .withUserConfiguration(CustomProcessorConfiguration::class.java) + .run { ctx -> + ctx.startupFailure shouldBe null + ctx.getBean(OutboxEntryProcessor::class.java).shouldNotBeNull() + ctx.getBean(OutboxProcessorScheduler::class.java).shouldNotBeNull() + } + } + test("properties are bound from application config") { contextRunner .withPropertyValues( @@ -184,6 +245,17 @@ class OutboxProcessorAutoConfigurationTest : FunSpec({ } } + test("duplicate deliverer types fail application startup") { + contextRunner + .withBean("secondDeliverer", MessageDeliverer::class.java, { stubDeliverer() }) + .run { ctx -> + val errors = generateSequence(ctx.startupFailure.shouldNotBeNull()) { it.cause } + .mapNotNull { it.message } + .joinToString(" ") + errors shouldContain "Duplicate MessageDeliverer type" + } + } + test("listener, metrics and refresher are wired when a MeterRegistry bean is provided directly") { contextRunner .withBean(io.micrometer.core.instrument.MeterRegistry::class.java, { @@ -287,7 +359,14 @@ class OutboxProcessorAutoConfigurationTest : FunSpec({ } }) -// Loads a Spring Boot auto-config class by trying version-specific FQCNs in order. +@Configuration(proxyBeanMethods = false) +private class CustomProcessorConfiguration { + @Bean + fun customProcessor(store: OutboxStore, entryProcessor: OutboxEntryProcessor): OutboxProcessor { + return OutboxProcessor(store, entryProcessor) + } +} + /** * Context with two thread-recording deliverers, so a batch spanning both transports proves where * `okapi.processor.transport-dispatch` actually lands: the recorded threads are the observable @@ -319,6 +398,7 @@ private fun entryOfType(t: String): OutboxEntry { return OutboxEntry.createPending(OutboxMessage("evt", "{}"), deliveryInfo, Instant.EPOCH) } +// Loads a Spring Boot auto-config class by trying version-specific FQCNs in order. // Lets a single test exercise both the 3.5.x (`...actuate.autoconfigure.metrics...`) and 4.0.x (`...micrometer.metrics.autoconfigure...`) layouts. private fun resolveSpringBootClass(vararg candidateFqcns: String): Class<*> { val classLoader = OkapiMicrometerAutoConfiguration::class.java.classLoader From a96e43ecb53d8a78df0e60d16490f7e6e70a0c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Kobyli=C5=84ski?= Date: Sat, 26 Sep 2026 12:15:39 +0200 Subject: [PATCH 2/5] fix: accept custom entry processor without deliverer bean --- .../okapi/core/CompositeMessageDeliverer.kt | 2 ++ .../okapi/core/OutboxEntryProcessor.kt | 3 +++ .../okapi/springboot/OutboxAutoConfiguration.kt | 6 +----- .../OutboxProcessorAutoConfigurationTest.kt | 15 +++++++++++++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/CompositeMessageDeliverer.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/CompositeMessageDeliverer.kt index d3bbc0e..b62bc0f 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/CompositeMessageDeliverer.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/CompositeMessageDeliverer.kt @@ -33,6 +33,8 @@ class CompositeMessageDeliverer @JvmOverloads constructor( } } + val supportedDeliveryTypes: Set = registry.keys + override fun deliver(entry: OutboxEntry): DeliveryResult { val messageDeliverer = registry[entry.deliveryType] ?: return DeliveryResult.PermanentFailure("No deliverer registered for type '${entry.deliveryType}'") diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntryProcessor.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntryProcessor.kt index 896abd9..52d2631 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntryProcessor.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntryProcessor.kt @@ -11,6 +11,9 @@ class OutboxEntryProcessor( private val retryPolicy: RetryPolicy, private val clock: Clock, ) { + val supportedDeliveryTypes: Set = + if (deliverer is CompositeMessageDeliverer) deliverer.supportedDeliveryTypes else setOf(deliverer.type) + fun process(entry: OutboxEntry): OutboxEntry = applyResult(entry, deliverer.deliver(entry), clock.instant()) /** diff --git a/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt b/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt index eb6be69..bb2a526 100644 --- a/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt +++ b/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt @@ -18,7 +18,6 @@ import org.springframework.beans.factory.BeanFactory import org.springframework.beans.factory.BeanNotOfRequiredTypeException import org.springframework.beans.factory.NoSuchBeanDefinitionException import org.springframework.beans.factory.ObjectProvider -import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.beans.factory.getBean import org.springframework.boot.autoconfigure.AutoConfiguration @@ -84,9 +83,6 @@ class OutboxAutoConfiguration( @field:Value("\${okapi.processor.enabled:true}") private var processorEnabled: Boolean = true - @field:Autowired - private lateinit var messageDeliverers: ObjectProvider - private fun resolveDataSource(): DataSource = resolveDataSource(dataSources, primaryDataSource, okapiProperties) @Bean @@ -193,7 +189,7 @@ class OutboxAutoConfiguration( listener: ObjectProvider, clock: ObjectProvider, ): OutboxProcessor { - check(!processorEnabled || messageDeliverers.iterator().hasNext()) { + check(!processorEnabled || outboxEntryProcessor.supportedDeliveryTypes.isNotEmpty()) { "Okapi processor is enabled but no MessageDeliverer bean is registered. " + "Define a MessageDeliverer or set okapi.processor.enabled=false for publisher-only use." } diff --git a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt index 726ce57..62a3bf8 100644 --- a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt +++ b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt @@ -105,6 +105,21 @@ class OutboxProcessorAutoConfigurationTest : FunSpec({ } } + test("custom entry processor starts without a MessageDeliverer bean") { + ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(OutboxAutoConfiguration::class.java)) + .withBean(OutboxStore::class.java, { stubStore() }) + .withBean(DataSource::class.java, { SimpleDriverDataSource() }) + .withBean(TransactionRunner::class.java, { noOpTransactionRunner() }) + .withBean(OutboxEntryProcessor::class.java, { + OutboxEntryProcessor(stubDeliverer(), RetryPolicy(maxRetries = 0), Clock.systemUTC()) + }) + .run { ctx -> + ctx.startupFailure shouldBe null + ctx.getBean(OutboxProcessorScheduler::class.java).shouldNotBeNull() + } + } + test("custom processor can use the auto-configured entry processor") { contextRunner .withUserConfiguration(CustomProcessorConfiguration::class.java) From a4b6894b218d304df1f5e35c1e5c53e47b3bdd04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Kobyli=C5=84ski?= Date: Sat, 26 Sep 2026 12:08:29 +0200 Subject: [PATCH 3/5] fix: claim only delivery types supported by each worker Add route-aware claims and a matching index for PostgreSQL and MySQL. Rotate the first route between batches, reject unsafe custom stores, and prevent a single deliverer from processing an entry of another type. Cover heterogeneous workers, concurrent claims, migrations, and a million-row Docker benchmark. Document the custom-store and rolling-upgrade requirements. Closes #116 --- README.md | 6 +- benchmarks/route-aware-claim/README.md | 23 ++++ .../okapi-116-my-client-100.sql | 101 ++++++++++++++++++ .../okapi-116-my-queries.sql | 6 ++ .../route-aware-claim/okapi-116-my-setup.sql | 30 ++++++ .../route-aware-claim/okapi-116-my-write.sql | 17 +++ .../okapi-116-pg-client-100.sql | 100 +++++++++++++++++ .../okapi-116-pg-queries.sql | 5 + .../route-aware-claim/okapi-116-pg-setup.sql | 22 ++++ .../route-aware-claim/okapi-116-pg-write.sql | 14 +++ .../okapi/core/OutboxEntryProcessor.kt | 10 +- .../okapi/core/OutboxProcessor.kt | 21 +++- .../softwaremill/okapi/core/OutboxStore.kt | 5 +- .../okapi/core/RouteAwareOutboxStore.kt | 11 ++ .../okapi/core/OutboxProcessorTest.kt | 9 +- .../okapi/core/OutboxSchedulerTest.kt | 6 +- .../core/RouteAwareOutboxProcessorTest.kt | 86 +++++++++++++++ .../test/concurrency/ConcurrentClaimTests.kt | 60 ++++++++++- .../test/store/OutboxStoreContractTests.kt | 22 ++++ .../okapi/mysql/MysqlOutboxStore.kt | 27 ++++- .../db/mysql/002__route_aware_claim_index.sql | 6 ++ .../softwaremill/okapi/db/mysql/changelog.xml | 1 + .../okapi/postgres/PostgresOutboxStore.kt | 25 ++++- .../postgres/002__route_aware_claim_index.sql | 5 + .../okapi/db/postgres/changelog.xml | 1 + .../springboot/OutboxAutoConfiguration.kt | 5 + .../okapi/springboot/OkapiSpringTestStubs.kt | 5 +- .../OutboxProcessorAutoConfigurationTest.kt | 14 +++ 28 files changed, 627 insertions(+), 16 deletions(-) create mode 100644 benchmarks/route-aware-claim/README.md create mode 100644 benchmarks/route-aware-claim/okapi-116-my-client-100.sql create mode 100644 benchmarks/route-aware-claim/okapi-116-my-queries.sql create mode 100644 benchmarks/route-aware-claim/okapi-116-my-setup.sql create mode 100644 benchmarks/route-aware-claim/okapi-116-my-write.sql create mode 100644 benchmarks/route-aware-claim/okapi-116-pg-client-100.sql create mode 100644 benchmarks/route-aware-claim/okapi-116-pg-queries.sql create mode 100644 benchmarks/route-aware-claim/okapi-116-pg-setup.sql create mode 100644 benchmarks/route-aware-claim/okapi-116-pg-write.sql create mode 100644 okapi-core/src/main/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxStore.kt create mode 100644 okapi-core/src/test/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxProcessorTest.kt create mode 100644 okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql create mode 100644 okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql diff --git a/README.md b/README.md index 25b7be0..8ee4061 100644 --- a/README.md +++ b/README.md @@ -85,14 +85,16 @@ Runnable, self-contained applications live in [okapi-examples](https://github.co ## How it works 1. `publish()` writes a `PENDING` row to `okapi_outbox` in your transaction. -2. A background scheduler polls for pending rows (every second by default), claiming them with `FOR UPDATE SKIP LOCKED` so workers do not process the same row concurrently. +2. A background scheduler polls for pending rows (every second by default), claiming only the delivery types configured on that worker with `FOR UPDATE SKIP LOCKED` so workers do not process the same row concurrently. 3. Each row goes to the transport matching its delivery type. Success marks it `DELIVERED`; a retriable failure leaves it `PENDING` while retry attempts remain; an exhausted retry budget or permanent failure marks it `FAILED`. 4. A purger deletes delivered rows after a retention period. ### Guarantees and limits - **Duplicate delivery is possible.** A crash between a successful delivery and the status update means the message may be sent again after restart. If processing a message more than once would cause unwanted effects, make the consumer idempotent — for example, deduplicate on a business key in the payload or a header set in the `DeliveryInfo`. okapi sends your payload and configured headers, but not the `OutboxId` returned by `publish()`; that identifier stays on the publisher side for correlation and logging. -- **Best-effort ordering.** Rows are claimed by `created_at`, oldest first. However, parallel delivery and retries mean messages may reach consumers in a different order. Strict delivery ordering is not guaranteed. +- **Best-effort ordering.** Rows are claimed by `created_at, id` within each delivery type. The processor rotates which type it checks first between batches. Parallel delivery and retries can also change arrival order. Strict delivery ordering across types or within a type is not guaranteed. +- **Different workers can handle different delivery types.** Each worker claims only its registered types. A custom `OutboxStore` used for processing must implement `RouteAwareOutboxStore`; otherwise processing fails before an unsafe claim. The older `claimPending(limit)` method remains available for callers that need it, but Okapi's processor does not use it. Apply the new index migration before starting updated workers if you manage the schema yourself. +- **Rolling upgrades need sequencing.** Workers running an older Okapi version can still claim rows of a newly introduced delivery type. Upgrade or stop all old processors before publishing that type. An upgrade does not automatically retry rows already marked `FAILED` by older workers. - **Failure classification is the transport's job.** Each deliverer decides what is retriable. HTTP: 5xx, 429, 408 and connection errors are retriable; other responses and TLS errors are permanent. Kafka: broker-side retriable exceptions are retried; authorization and configuration errors are not. - **Retry budget.** `okapi.processor.max-retries` (default 5) counts retries *after* the first attempt — six attempts in total before a row becomes `FAILED`. `FAILED` is terminal. Retriable messages become eligible again on the next processor poll; there is no per-message backoff. diff --git a/benchmarks/route-aware-claim/README.md b/benchmarks/route-aware-claim/README.md new file mode 100644 index 0000000..5fc2c06 --- /dev/null +++ b/benchmarks/route-aware-claim/README.md @@ -0,0 +1,23 @@ +# Route-aware claiming on a large outbox + +Local exploratory measurements with PostgreSQL 16 and MySQL 8.0 in Docker. The setup SQL files create a table with the Okapi columns and 1,000,000 `PENDING` rows. The oldest 900,000 use `delivery_type='unhandled'`; the newer 100,000 are split evenly across 100 `handled_00` through `handled_99` routes. Both the old `(status, created_at)` index and the new `(status, delivery_type, created_at, id)` index are present. + +Run `okapi-116-pg-setup.sql` with `psql` or `okapi-116-my-setup.sql` with `mysql`, then run the matching `okapi-116-*-queries.sql` file. The query files compare one route, an `IN` query for 100 present routes, and an `IN` query for 100 absent routes. All use `FOR UPDATE SKIP LOCKED` and a batch limit of 100. The `okapi-116-*-client-100.sql` files send 100 separate empty claims through a SQL client. + +| Case | PostgreSQL observed execution | MySQL observed execution | +| --- | ---: | ---: | +| One present route, first 100 rows | 0.12 ms | 0.49 ms | +| `IN` with 100 absent routes, ordered by type and creation time | 165 ms | 0.21 ms | +| 100 separate empty route claims, server-side loop | 1.57 ms | 2.04–2.37 ms | +| 100 separate empty route claims through one local SQL client process | 24 ms | 15 ms | + +The PostgreSQL plan for the absent `IN` case scanned all 1,000,000 rows and removed them by filter, despite the new index. The one-route plan on both databases used the new composite index and stopped at the limit. This is why the processor claims one route at a time and stops once its batch is full. At 100 empty routes it still performs 100 small queries per poll; this cost grows with the number of configured routes. + +The `okapi-116-*-write.sql` files insert 20,000 rows and update about 20,000 statuses inside rolled-back transactions. One run with the new index and one after dropping only that index gave: + +| Operation | PostgreSQL with / without new index | MySQL with / without new index | +| --- | ---: | ---: | +| Insert 20,000 | 125 / 81 ms | 177 / 141 ms | +| Update about 20,000 statuses | 158 / 135 ms | 363 / 304 ms | + +These write figures are **single local runs**, sensitive to cache state and test order. They show a real write-cost tradeoff, not a stable throughput estimate. The read measurements are also exploratory; the query plan and scanned-row counts are the main evidence. Integration tests separately cover route isolation, concurrent processors, migrations, and delivery behavior on both databases. diff --git a/benchmarks/route-aware-claim/okapi-116-my-client-100.sql b/benchmarks/route-aware-claim/okapi-116-my-client-100.sql new file mode 100644 index 0000000..cf93a85 --- /dev/null +++ b/benchmarks/route-aware-claim/okapi-116-my-client-100.sql @@ -0,0 +1,101 @@ +USE okapi_bench; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_00' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_01' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_02' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_03' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_04' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_05' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_06' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_07' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_08' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_09' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_10' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_11' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_12' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_13' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_14' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_15' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_16' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_17' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_18' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_19' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_20' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_21' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_22' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_23' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_24' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_25' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_26' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_27' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_28' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_29' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_30' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_31' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_32' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_33' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_34' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_35' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_36' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_37' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_38' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_39' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_40' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_41' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_42' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_43' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_44' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_45' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_46' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_47' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_48' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_49' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_50' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_51' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_52' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_53' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_54' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_55' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_56' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_57' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_58' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_59' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_60' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_61' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_62' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_63' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_64' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_65' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_66' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_67' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_68' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_69' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_70' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_71' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_72' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_73' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_74' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_75' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_76' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_77' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_78' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_79' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_80' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_81' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_82' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_83' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_84' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_85' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_86' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_87' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_88' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_89' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_90' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_91' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_92' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_93' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_94' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_95' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_96' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_97' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_98' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='absent_99' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; diff --git a/benchmarks/route-aware-claim/okapi-116-my-queries.sql b/benchmarks/route-aware-claim/okapi-116-my-queries.sql new file mode 100644 index 0000000..d51c030 --- /dev/null +++ b/benchmarks/route-aware-claim/okapi-116-my-queries.sql @@ -0,0 +1,6 @@ +USE okapi_bench; +START TRANSACTION; +EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='handled_00' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type IN ('handled_00', 'handled_01', 'handled_02', 'handled_03', 'handled_04', 'handled_05', 'handled_06', 'handled_07', 'handled_08', 'handled_09', 'handled_10', 'handled_11', 'handled_12', 'handled_13', 'handled_14', 'handled_15', 'handled_16', 'handled_17', 'handled_18', 'handled_19', 'handled_20', 'handled_21', 'handled_22', 'handled_23', 'handled_24', 'handled_25', 'handled_26', 'handled_27', 'handled_28', 'handled_29', 'handled_30', 'handled_31', 'handled_32', 'handled_33', 'handled_34', 'handled_35', 'handled_36', 'handled_37', 'handled_38', 'handled_39', 'handled_40', 'handled_41', 'handled_42', 'handled_43', 'handled_44', 'handled_45', 'handled_46', 'handled_47', 'handled_48', 'handled_49', 'handled_50', 'handled_51', 'handled_52', 'handled_53', 'handled_54', 'handled_55', 'handled_56', 'handled_57', 'handled_58', 'handled_59', 'handled_60', 'handled_61', 'handled_62', 'handled_63', 'handled_64', 'handled_65', 'handled_66', 'handled_67', 'handled_68', 'handled_69', 'handled_70', 'handled_71', 'handled_72', 'handled_73', 'handled_74', 'handled_75', 'handled_76', 'handled_77', 'handled_78', 'handled_79', 'handled_80', 'handled_81', 'handled_82', 'handled_83', 'handled_84', 'handled_85', 'handled_86', 'handled_87', 'handled_88', 'handled_89', 'handled_90', 'handled_91', 'handled_92', 'handled_93', 'handled_94', 'handled_95', 'handled_96', 'handled_97', 'handled_98', 'handled_99') ORDER BY delivery_type,created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type IN ('absent_00', 'absent_01', 'absent_02', 'absent_03', 'absent_04', 'absent_05', 'absent_06', 'absent_07', 'absent_08', 'absent_09', 'absent_10', 'absent_11', 'absent_12', 'absent_13', 'absent_14', 'absent_15', 'absent_16', 'absent_17', 'absent_18', 'absent_19', 'absent_20', 'absent_21', 'absent_22', 'absent_23', 'absent_24', 'absent_25', 'absent_26', 'absent_27', 'absent_28', 'absent_29', 'absent_30', 'absent_31', 'absent_32', 'absent_33', 'absent_34', 'absent_35', 'absent_36', 'absent_37', 'absent_38', 'absent_39', 'absent_40', 'absent_41', 'absent_42', 'absent_43', 'absent_44', 'absent_45', 'absent_46', 'absent_47', 'absent_48', 'absent_49', 'absent_50', 'absent_51', 'absent_52', 'absent_53', 'absent_54', 'absent_55', 'absent_56', 'absent_57', 'absent_58', 'absent_59', 'absent_60', 'absent_61', 'absent_62', 'absent_63', 'absent_64', 'absent_65', 'absent_66', 'absent_67', 'absent_68', 'absent_69', 'absent_70', 'absent_71', 'absent_72', 'absent_73', 'absent_74', 'absent_75', 'absent_76', 'absent_77', 'absent_78', 'absent_79', 'absent_80', 'absent_81', 'absent_82', 'absent_83', 'absent_84', 'absent_85', 'absent_86', 'absent_87', 'absent_88', 'absent_89', 'absent_90', 'absent_91', 'absent_92', 'absent_93', 'absent_94', 'absent_95', 'absent_96', 'absent_97', 'absent_98', 'absent_99') ORDER BY delivery_type,created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +ROLLBACK; diff --git a/benchmarks/route-aware-claim/okapi-116-my-setup.sql b/benchmarks/route-aware-claim/okapi-116-my-setup.sql new file mode 100644 index 0000000..d3ed91c --- /dev/null +++ b/benchmarks/route-aware-claim/okapi-116-my-setup.sql @@ -0,0 +1,30 @@ +CREATE DATABASE IF NOT EXISTS okapi_bench; +USE okapi_bench; +CREATE TABLE IF NOT EXISTS okapi_outbox ( + id CHAR(36) NOT NULL PRIMARY KEY, + message_type VARCHAR(255) NOT NULL, + payload TEXT NOT NULL, + delivery_type VARCHAR(50) NOT NULL, + status VARCHAR(50) NOT NULL DEFAULT 'PENDING', + created_at TIMESTAMP(6) NOT NULL, + updated_at TIMESTAMP(6) NOT NULL, + retries INT NOT NULL DEFAULT 0, + last_attempt TIMESTAMP(6) NULL, + last_error TEXT, + delivery_metadata JSON NOT NULL +); +CREATE TABLE digit (n INT NOT NULL); +INSERT INTO digit VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9); +INSERT INTO okapi_outbox +SELECT LPAD(x.n, 36, '0'), 'event', '{}', + CASE WHEN x.n <= 900000 THEN 'unhandled' ELSE CONCAT('handled_', LPAD(FLOOR((x.n - 900001) / 1000), 2, '0')) END, + 'PENDING', TIMESTAMPADD(MICROSECOND, x.n, '2024-01-01 00:00:00'), + '2024-01-01 00:00:00', 0, NULL, NULL, JSON_OBJECT() +FROM ( + SELECT a.n + 10*b.n + 100*c.n + 1000*d.n + 10000*e.n + 100000*f.n + 1 AS n + FROM digit a CROSS JOIN digit b CROSS JOIN digit c CROSS JOIN digit d CROSS JOIN digit e CROSS JOIN digit f +) x; +CREATE INDEX idx_okapi_outbox_status_created_at ON okapi_outbox (status, created_at); +CREATE INDEX idx_okapi_outbox_status_delivery_created_id ON okapi_outbox (status, delivery_type, created_at, id); +ANALYZE TABLE okapi_outbox; +DROP TABLE digit; diff --git a/benchmarks/route-aware-claim/okapi-116-my-write.sql b/benchmarks/route-aware-claim/okapi-116-my-write.sql new file mode 100644 index 0000000..193f815 --- /dev/null +++ b/benchmarks/route-aware-claim/okapi-116-my-write.sql @@ -0,0 +1,17 @@ +USE okapi_bench; +START TRANSACTION; +SET @started = NOW(6); +INSERT INTO okapi_outbox +SELECT CONCAT('b', SUBSTRING(id, 2)), message_type, payload, delivery_type, status, + created_at, updated_at, retries, last_attempt, last_error, delivery_metadata +FROM okapi_outbox +WHERE id <= LPAD(20000, 36, '0') +ORDER BY id LIMIT 20000; +SELECT TIMESTAMPDIFF(MICROSECOND, @started, NOW(6)) / 1000 AS insert_20k_ms; +ROLLBACK; +START TRANSACTION; +SET @started = NOW(6); +UPDATE okapi_outbox SET status = 'DELIVERED' +WHERE id <= LPAD(20000, 36, '0'); +SELECT TIMESTAMPDIFF(MICROSECOND, @started, NOW(6)) / 1000 AS update_20k_ms; +ROLLBACK; diff --git a/benchmarks/route-aware-claim/okapi-116-pg-client-100.sql b/benchmarks/route-aware-claim/okapi-116-pg-client-100.sql new file mode 100644 index 0000000..2de79c7 --- /dev/null +++ b/benchmarks/route-aware-claim/okapi-116-pg-client-100.sql @@ -0,0 +1,100 @@ +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_00' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_01' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_02' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_03' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_04' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_05' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_06' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_07' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_08' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_09' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_10' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_11' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_12' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_13' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_14' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_15' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_16' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_17' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_18' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_19' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_20' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_21' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_22' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_23' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_24' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_25' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_26' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_27' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_28' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_29' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_30' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_31' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_32' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_33' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_34' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_35' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_36' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_37' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_38' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_39' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_40' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_41' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_42' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_43' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_44' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_45' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_46' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_47' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_48' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_49' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_50' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_51' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_52' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_53' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_54' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_55' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_56' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_57' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_58' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_59' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_60' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_61' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_62' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_63' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_64' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_65' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_66' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_67' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_68' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_69' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_70' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_71' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_72' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_73' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_74' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_75' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_76' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_77' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_78' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_79' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_80' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_81' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_82' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_83' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_84' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_85' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_86' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_87' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_88' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_89' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_90' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_91' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_92' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_93' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_94' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_95' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_96' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_97' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_98' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +SELECT id FROM okapi_outbox WHERE status='PENDING' AND delivery_type='absent_99' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; diff --git a/benchmarks/route-aware-claim/okapi-116-pg-queries.sql b/benchmarks/route-aware-claim/okapi-116-pg-queries.sql new file mode 100644 index 0000000..3c2f755 --- /dev/null +++ b/benchmarks/route-aware-claim/okapi-116-pg-queries.sql @@ -0,0 +1,5 @@ +BEGIN; +EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM okapi_outbox WHERE status='PENDING' AND delivery_type='handled_00' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM okapi_outbox WHERE status='PENDING' AND delivery_type IN ('handled_00', 'handled_01', 'handled_02', 'handled_03', 'handled_04', 'handled_05', 'handled_06', 'handled_07', 'handled_08', 'handled_09', 'handled_10', 'handled_11', 'handled_12', 'handled_13', 'handled_14', 'handled_15', 'handled_16', 'handled_17', 'handled_18', 'handled_19', 'handled_20', 'handled_21', 'handled_22', 'handled_23', 'handled_24', 'handled_25', 'handled_26', 'handled_27', 'handled_28', 'handled_29', 'handled_30', 'handled_31', 'handled_32', 'handled_33', 'handled_34', 'handled_35', 'handled_36', 'handled_37', 'handled_38', 'handled_39', 'handled_40', 'handled_41', 'handled_42', 'handled_43', 'handled_44', 'handled_45', 'handled_46', 'handled_47', 'handled_48', 'handled_49', 'handled_50', 'handled_51', 'handled_52', 'handled_53', 'handled_54', 'handled_55', 'handled_56', 'handled_57', 'handled_58', 'handled_59', 'handled_60', 'handled_61', 'handled_62', 'handled_63', 'handled_64', 'handled_65', 'handled_66', 'handled_67', 'handled_68', 'handled_69', 'handled_70', 'handled_71', 'handled_72', 'handled_73', 'handled_74', 'handled_75', 'handled_76', 'handled_77', 'handled_78', 'handled_79', 'handled_80', 'handled_81', 'handled_82', 'handled_83', 'handled_84', 'handled_85', 'handled_86', 'handled_87', 'handled_88', 'handled_89', 'handled_90', 'handled_91', 'handled_92', 'handled_93', 'handled_94', 'handled_95', 'handled_96', 'handled_97', 'handled_98', 'handled_99') ORDER BY delivery_type,created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +EXPLAIN (ANALYZE, BUFFERS) SELECT * FROM okapi_outbox WHERE status='PENDING' AND delivery_type IN ('absent_00', 'absent_01', 'absent_02', 'absent_03', 'absent_04', 'absent_05', 'absent_06', 'absent_07', 'absent_08', 'absent_09', 'absent_10', 'absent_11', 'absent_12', 'absent_13', 'absent_14', 'absent_15', 'absent_16', 'absent_17', 'absent_18', 'absent_19', 'absent_20', 'absent_21', 'absent_22', 'absent_23', 'absent_24', 'absent_25', 'absent_26', 'absent_27', 'absent_28', 'absent_29', 'absent_30', 'absent_31', 'absent_32', 'absent_33', 'absent_34', 'absent_35', 'absent_36', 'absent_37', 'absent_38', 'absent_39', 'absent_40', 'absent_41', 'absent_42', 'absent_43', 'absent_44', 'absent_45', 'absent_46', 'absent_47', 'absent_48', 'absent_49', 'absent_50', 'absent_51', 'absent_52', 'absent_53', 'absent_54', 'absent_55', 'absent_56', 'absent_57', 'absent_58', 'absent_59', 'absent_60', 'absent_61', 'absent_62', 'absent_63', 'absent_64', 'absent_65', 'absent_66', 'absent_67', 'absent_68', 'absent_69', 'absent_70', 'absent_71', 'absent_72', 'absent_73', 'absent_74', 'absent_75', 'absent_76', 'absent_77', 'absent_78', 'absent_79', 'absent_80', 'absent_81', 'absent_82', 'absent_83', 'absent_84', 'absent_85', 'absent_86', 'absent_87', 'absent_88', 'absent_89', 'absent_90', 'absent_91', 'absent_92', 'absent_93', 'absent_94', 'absent_95', 'absent_96', 'absent_97', 'absent_98', 'absent_99') ORDER BY delivery_type,created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +ROLLBACK; diff --git a/benchmarks/route-aware-claim/okapi-116-pg-setup.sql b/benchmarks/route-aware-claim/okapi-116-pg-setup.sql new file mode 100644 index 0000000..4ef042a --- /dev/null +++ b/benchmarks/route-aware-claim/okapi-116-pg-setup.sql @@ -0,0 +1,22 @@ +CREATE TABLE okapi_outbox ( + id UUID NOT NULL PRIMARY KEY, + message_type VARCHAR(255) NOT NULL, + payload TEXT NOT NULL, + delivery_type VARCHAR(50) NOT NULL, + status VARCHAR(50) NOT NULL DEFAULT 'PENDING', + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + retries INT NOT NULL DEFAULT 0, + last_attempt TIMESTAMP, + last_error TEXT, + delivery_metadata JSONB NOT NULL +); +INSERT INTO okapi_outbox +SELECT md5(n::text)::uuid, 'event', '{}', + CASE WHEN n <= 900000 THEN 'unhandled' ELSE 'handled_' || lpad(((n - 900001) / 1000)::text, 2, '0') END, + 'PENDING', timestamp '2024-01-01' + n * interval '1 microsecond', + timestamp '2024-01-01', 0, NULL, NULL, '{}'::jsonb +FROM generate_series(1, 1000000) AS n; +CREATE INDEX idx_okapi_outbox_status_created_at ON okapi_outbox (status, created_at); +CREATE INDEX idx_okapi_outbox_status_delivery_created_id ON okapi_outbox (status, delivery_type, created_at, id); +ANALYZE okapi_outbox; diff --git a/benchmarks/route-aware-claim/okapi-116-pg-write.sql b/benchmarks/route-aware-claim/okapi-116-pg-write.sql new file mode 100644 index 0000000..625eba3 --- /dev/null +++ b/benchmarks/route-aware-claim/okapi-116-pg-write.sql @@ -0,0 +1,14 @@ +BEGIN; +EXPLAIN (ANALYZE, BUFFERS) +INSERT INTO okapi_outbox +SELECT md5(('bench' || n)::text)::uuid, 'event', '{}', 'handled_00', 'PENDING', + timestamp '2024-02-01' + n * interval '1 microsecond', + timestamp '2024-02-01', 0, NULL, NULL, '{}'::jsonb +FROM generate_series(1, 20000) AS n; +ROLLBACK; +BEGIN; +EXPLAIN (ANALYZE, BUFFERS) +UPDATE okapi_outbox SET status = 'DELIVERED' +WHERE id >= '00000000-0000-0000-0000-000000000000'::uuid + AND id < '05000000-0000-0000-0000-000000000000'::uuid; +ROLLBACK; diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntryProcessor.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntryProcessor.kt index 52d2631..fbf00c4 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntryProcessor.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntryProcessor.kt @@ -14,7 +14,12 @@ class OutboxEntryProcessor( val supportedDeliveryTypes: Set = if (deliverer is CompositeMessageDeliverer) deliverer.supportedDeliveryTypes else setOf(deliverer.type) - fun process(entry: OutboxEntry): OutboxEntry = applyResult(entry, deliverer.deliver(entry), clock.instant()) + fun process(entry: OutboxEntry): OutboxEntry { + check(entry.deliveryType in supportedDeliveryTypes) { + "No deliverer registered for type '${entry.deliveryType}'" + } + return applyResult(entry, deliverer.deliver(entry), clock.instant()) + } /** * Processes a batch of entries via [MessageDeliverer.deliverBatch], applying @@ -23,6 +28,9 @@ class OutboxEntryProcessor( */ fun processBatch(entries: List): List { if (entries.isEmpty()) return emptyList() + check(entries.all { it.deliveryType in supportedDeliveryTypes }) { + "Batch contains an entry with no registered deliverer" + } val now = clock.instant() return deliverer.deliverBatch(entries).map { (entry, result) -> applyResult(entry, result, now) } } diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt index d942b33..a57ae78 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt @@ -3,6 +3,7 @@ package com.softwaremill.okapi.core import org.slf4j.LoggerFactory import java.time.Clock import java.time.Duration +import java.util.concurrent.atomic.AtomicInteger /** * Orchestrates a single processing cycle: claims pending entries from [OutboxStore], @@ -24,6 +25,9 @@ class OutboxProcessor @JvmOverloads constructor( private val listener: OutboxProcessorListener? = null, private val clock: Clock = Clock.systemUTC(), ) { + private val deliveryTypes = entryProcessor.supportedDeliveryTypes.toList() + private val nextStartingType = AtomicInteger() + /** * Claims up to [limit] pending entries, processes them as a batch, and persists * each result. Returns the number of entries processed (0 if the store had nothing). @@ -31,7 +35,22 @@ class OutboxProcessor @JvmOverloads constructor( @JvmOverloads fun processNext(limit: Int = 10): Int { val batchStart = clock.instant() - val claimed = store.claimPending(limit) + val routeAwareStore = store as? RouteAwareOutboxStore + ?: error("OutboxProcessor requires a RouteAwareOutboxStore to avoid claiming unsupported delivery types") + check(deliveryTypes.isNotEmpty()) { "OutboxProcessor requires at least one delivery type" } + val claimed = buildList { + val start = Math.floorMod(nextStartingType.getAndIncrement(), deliveryTypes.size) + for (offset in deliveryTypes.indices) { + val remaining = limit - size + if (remaining <= 0) break + val deliveryType = deliveryTypes[(start + offset) % deliveryTypes.size] + val entries = routeAwareStore.claimPending(deliveryType, remaining) + check(entries.size <= remaining && entries.all { it.deliveryType == deliveryType }) { + "RouteAwareOutboxStore returned entries outside the requested type or limit" + } + addAll(entries) + } + } if (claimed.isEmpty()) { notifyBatch(0, Duration.between(batchStart, clock.instant())) return 0 diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxStore.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxStore.kt index 8b21a11..87068d3 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxStore.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxStore.kt @@ -6,7 +6,10 @@ interface OutboxStore { /** Persists a new outbox entry (publish). */ fun persist(entry: OutboxEntry): OutboxEntry - /** Claims up to [limit] PENDING entries with exclusive access. Locking strategy depends on the adapter. */ + /** + * Claims up to [limit] PENDING entries without filtering by delivery type. + * Kept for direct callers; [OutboxProcessor] uses [RouteAwareOutboxStore] instead. + */ fun claimPending(limit: Int): List /** Updates an entry after a delivery attempt (status change, retries, lastError). */ diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxStore.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxStore.kt new file mode 100644 index 0000000..08483b4 --- /dev/null +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxStore.kt @@ -0,0 +1,11 @@ +package com.softwaremill.okapi.core + +/** + * Claims only entries for a delivery type handled by the current worker. + * The returned rows must be locked until the processing transaction completes. + * Implementations should return at most the requested number of rows ordered by creation time and ID. + * The processor rejects results with another [OutboxEntry.deliveryType] or more than requested. + */ +interface RouteAwareOutboxStore : OutboxStore { + fun claimPending(deliveryType: String, limit: Int): List +} diff --git a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxProcessorTest.kt b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxProcessorTest.kt index 5668f7c..effda8f 100644 --- a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxProcessorTest.kt +++ b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxProcessorTest.kt @@ -30,11 +30,14 @@ class OutboxProcessorTest : var pendingEntries: List = emptyList() val store = - object : OutboxStore { + object : RouteAwareOutboxStore { override fun persist(entry: OutboxEntry): OutboxEntry = entry override fun claimPending(limit: Int): List = pendingEntries.take(limit) + override fun claimPending(deliveryType: String, limit: Int): List = + pendingEntries.filter { it.deliveryType == deliveryType }.take(limit) + override fun updateAfterProcessing(entry: OutboxEntry): OutboxEntry = entry.also { processedEntries += it } override fun removeDeliveredBefore(time: Instant, limit: Int): Int = 0 @@ -334,9 +337,11 @@ class OutboxProcessorTest : `when`("store overrides updateAfterProcessingBatch") { val batchCalls = mutableListOf>() val individualCalls = mutableListOf() - val batchAwareStore = object : OutboxStore { + val batchAwareStore = object : RouteAwareOutboxStore { override fun persist(entry: OutboxEntry) = entry override fun claimPending(limit: Int): List = pendingEntries.take(limit) + override fun claimPending(deliveryType: String, limit: Int): List = + pendingEntries.filter { it.deliveryType == deliveryType }.take(limit) override fun updateAfterProcessing(entry: OutboxEntry): OutboxEntry = entry.also { individualCalls += it } override fun removeDeliveredBefore(time: Instant, limit: Int) = 0 override fun findOldestCreatedAt(statuses: Set) = emptyMap() diff --git a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxSchedulerTest.kt b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxSchedulerTest.kt index 84543e2..7ee10d9 100644 --- a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxSchedulerTest.kt +++ b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxSchedulerTest.kt @@ -686,12 +686,16 @@ private fun awaitTermination(thread: Thread, timeoutMs: Long = 5_000) { } private fun stubProcessor(onProcessNext: (Int) -> Unit): OutboxProcessor { - val store = object : OutboxStore { + val store = object : RouteAwareOutboxStore { override fun persist(entry: OutboxEntry) = entry override fun claimPending(limit: Int): List { onProcessNext(limit) return emptyList() } + override fun claimPending(deliveryType: String, limit: Int): List { + onProcessNext(limit) + return emptyList() + } override fun updateAfterProcessing(entry: OutboxEntry) = entry override fun removeDeliveredBefore(time: java.time.Instant, limit: Int) = 0 override fun findOldestCreatedAt(statuses: Set) = emptyMap() diff --git a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxProcessorTest.kt b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxProcessorTest.kt new file mode 100644 index 0000000..ebfb185 --- /dev/null +++ b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxProcessorTest.kt @@ -0,0 +1,86 @@ +package com.softwaremill.okapi.core + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.shouldBe +import java.time.Clock +import java.time.Instant + +private fun routeEntry(type: String, seconds: Long = 0): OutboxEntry = OutboxEntry.createPending( + OutboxMessage("event", "{}"), + object : DeliveryInfo { + override val type = type + override fun serialize() = "{}" + }, + Instant.EPOCH.plusSeconds(seconds), +) + +private fun routeDeliverer(type: String) = object : MessageDeliverer { + override val type = type + override fun deliver(entry: OutboxEntry) = DeliveryResult.Success +} + +private class RoutingStore(var pending: List) : RouteAwareOutboxStore { + val queriedTypes = mutableListOf() + val processed = mutableListOf() + + override fun persist(entry: OutboxEntry) = entry + override fun claimPending(limit: Int): List = error("Unsafe claimPending(limit) must not be used") + override fun claimPending(deliveryType: String, limit: Int): List { + queriedTypes += deliveryType + val claimed = pending.filter { it.deliveryType == deliveryType }.take(limit) + pending = pending - claimed.toSet() + return claimed + } + override fun updateAfterProcessing(entry: OutboxEntry) = entry.also { processed += it } + override fun removeDeliveredBefore(time: Instant, limit: Int) = 0 + override fun findOldestCreatedAt(statuses: Set) = emptyMap() + override fun countByStatuses() = emptyMap() +} + +class RouteAwareOutboxProcessorTest : FunSpec({ + test("a single deliverer refuses an entry of another type before delivery") { + val entryProcessor = OutboxEntryProcessor(routeDeliverer("kafka"), RetryPolicy(1), Clock.systemUTC()) + + shouldThrow { entryProcessor.process(routeEntry("http")) } + } + + test("older unsupported entries are never claimed or failed") { + val unsupported = routeEntry("http") + val supported = routeEntry("kafka", 1) + val store = RoutingStore(listOf(unsupported, supported)) + val processor = OutboxProcessor( + store, + OutboxEntryProcessor(routeDeliverer("kafka"), RetryPolicy(1), Clock.systemUTC()), + ) + + processor.processNext(10) shouldBe 1 + store.pending shouldContainExactly listOf(unsupported) + store.processed.single().status shouldBe OutboxStatus.DELIVERED + store.queriedTypes shouldContainExactly listOf("kafka") + } + + test("starting route rotates and claims at most the batch limit") { + val store = RoutingStore(listOf(routeEntry("kafka"), routeEntry("http"))) + val deliverer = CompositeMessageDeliverer(listOf(routeDeliverer("kafka"), routeDeliverer("http"))) + val processor = OutboxProcessor(store, OutboxEntryProcessor(deliverer, RetryPolicy(1), Clock.systemUTC())) + + processor.processNext(1) shouldBe 1 + processor.processNext(1) shouldBe 1 + store.queriedTypes shouldContainExactly listOf("kafka", "http") + store.processed.map { it.deliveryType } shouldContainExactly listOf("kafka", "http") + } + + test("processing with a store lacking route-aware claims fails before legacy claim") { + val store = object : OutboxStore by RoutingStore(emptyList()) { + override fun claimPending(limit: Int): List = error("legacy claim was called") + } + val processor = OutboxProcessor( + store, + OutboxEntryProcessor(routeDeliverer("kafka"), RetryPolicy(1), Clock.systemUTC()), + ) + + shouldThrow { processor.processNext() } + } +}) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/concurrency/ConcurrentClaimTests.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/concurrency/ConcurrentClaimTests.kt index 73142d2..d60dd4e 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/concurrency/ConcurrentClaimTests.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/concurrency/ConcurrentClaimTests.kt @@ -1,6 +1,8 @@ package com.softwaremill.okapi.test.concurrency import com.softwaremill.okapi.core.DeliveryInfo +import com.softwaremill.okapi.core.DeliveryResult +import com.softwaremill.okapi.core.MessageDeliverer import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxEntryProcessor import com.softwaremill.okapi.core.OutboxId @@ -29,6 +31,7 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.CyclicBarrier import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger private class StubDeliveryInfo( override val type: String = "recording", @@ -37,9 +40,13 @@ private class StubDeliveryInfo( override fun serialize(): String = metadata } -private fun createTestEntry(index: Int, now: Instant = Instant.parse("2024-01-01T00:00:00Z")): OutboxEntry = OutboxEntry.createPending( +private fun createTestEntry( + index: Int, + now: Instant = Instant.parse("2024-01-01T00:00:00Z"), + deliveryType: String = "recording", +): OutboxEntry = OutboxEntry.createPending( message = OutboxMessage(messageType = "concurrent.test", payload = """{"index":$index}"""), - deliveryInfo = StubDeliveryInfo(), + deliveryInfo = StubDeliveryInfo(deliveryType), now = now.plusSeconds(index.toLong()), ) @@ -170,6 +177,55 @@ fun FunSpec.concurrentClaimTests( } } + test("[$dbName] workers with different routes process only their own entries") { + jdbc.withTransaction { + (0 until 20).forEach { i -> + store.persist(createTestEntry(i, deliveryType = "kafka")) + store.persist(createTestEntry(i, deliveryType = "http")) + } + } + val kafkaCount = AtomicInteger() + val httpCount = AtomicInteger() + val barrier = CyclicBarrier(2) + + fun processor(type: String, count: AtomicInteger): OutboxProcessor { + val deliverer = object : MessageDeliverer { + override val type = type + override fun deliver(entry: OutboxEntry): DeliveryResult { + count.incrementAndGet() + return DeliveryResult.Success + } + } + return OutboxProcessor(store, OutboxEntryProcessor(deliverer, RetryPolicy(0), Clock.systemUTC())) + } + + val kafka = processor("kafka", kafkaCount) + val http = processor("http", httpCount) + val executor = Executors.newVirtualThreadPerTaskExecutor() + try { + val tasks = listOf(kafka, http).map { worker -> + CompletableFuture.supplyAsync( + { + barrier.await(10, TimeUnit.SECONDS) + jdbc.withTransaction(transactionIsolation = Connection.TRANSACTION_READ_COMMITTED) { + worker.processNext(20) + } + }, + executor, + ) + } + tasks.forEach { it.get(30, TimeUnit.SECONDS) shouldBe 20 } + } finally { + executor.shutdown() + } + + kafkaCount.get() shouldBe 20 + httpCount.get() shouldBe 20 + val counts = jdbc.withTransaction { store.countByStatuses() } + counts shouldContain (OutboxStatus.DELIVERED to 40L) + counts shouldContain (OutboxStatus.FAILED to 0L) + } + test("[$dbName] OutboxScheduler with concurrency=4 workers processes disjoint batches without amplification") { val fixedClock = Clock.fixed(Instant.parse("2024-01-01T00:00:00Z"), ZoneOffset.UTC) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/OutboxStoreContractTests.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/OutboxStoreContractTests.kt index 02d1394..bee7d43 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/OutboxStoreContractTests.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/OutboxStoreContractTests.kt @@ -5,6 +5,7 @@ import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxMessage import com.softwaremill.okapi.core.OutboxStatus import com.softwaremill.okapi.core.OutboxStore +import com.softwaremill.okapi.core.RouteAwareOutboxStore import com.softwaremill.okapi.test.support.JdbcConnectionProvider import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.collections.shouldHaveSize @@ -73,6 +74,27 @@ fun FunSpec.outboxStoreContractTests( found.deliveryMetadata.replace(" ", "") shouldBe """{"type":"test"}""" } + test("[$dbName] route-aware claim skips older entries of unsupported types") { + val unsupported = createTestEntry( + now = Instant.parse("2024-01-01T00:00:00Z"), + deliveryInfo = StubDeliveryInfo("http"), + ) + val supported = createTestEntry( + now = Instant.parse("2024-01-02T00:00:00Z"), + deliveryInfo = StubDeliveryInfo("kafka"), + ) + jdbc.withTransaction { + store.persist(unsupported) + store.persist(supported) + } + + val routed = store as RouteAwareOutboxStore + val claimed = jdbc.withTransaction { routed.claimPending("kafka", 10) } + + claimed.map { it.outboxId } shouldBe listOf(supported.outboxId) + jdbc.withTransaction { routed.claimPending("http", 10) }.map { it.outboxId } shouldBe listOf(unsupported.outboxId) + } + test("[$dbName] claimPending returns entries ordered by created_at ASC") { val t1 = Instant.parse("2024-01-01T00:00:00Z") val t2 = Instant.parse("2024-01-02T00:00:00Z") diff --git a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt index 38a84bd..8803ade 100644 --- a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt +++ b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt @@ -4,7 +4,7 @@ import com.softwaremill.okapi.core.ConnectionProvider import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxId import com.softwaremill.okapi.core.OutboxStatus -import com.softwaremill.okapi.core.OutboxStore +import com.softwaremill.okapi.core.RouteAwareOutboxStore import java.sql.PreparedStatement import java.sql.ResultSet import java.sql.Timestamp @@ -15,7 +15,7 @@ import java.util.UUID /** MySQL [OutboxStore] implementation using plain JDBC. */ class MysqlOutboxStore( private val connectionProvider: ConnectionProvider, -) : OutboxStore { +) : RouteAwareOutboxStore { override fun persist(entry: OutboxEntry): OutboxEntry { connectionProvider.withConnection { conn -> @@ -68,6 +68,29 @@ class MysqlOutboxStore( } } + override fun claimPending(deliveryType: String, limit: Int): List { + // A matching index avoids scanning and locking rows for other delivery types. + val sql = """ + SELECT * FROM okapi_outbox + FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) + WHERE status = ? AND delivery_type = ? + ORDER BY created_at ASC, id ASC + LIMIT ? + FOR UPDATE SKIP LOCKED + """.trimIndent() + + return connectionProvider.withConnection { conn -> + conn.prepareStatement(sql).use { stmt -> + stmt.setString(1, OutboxStatus.PENDING.name) + stmt.setString(2, deliveryType) + stmt.setInt(3, limit) + stmt.executeQuery().use { rs -> + generateSequence { if (rs.next()) rs.toOutboxEntry() else null }.toList() + } + } + } + } + override fun updateAfterProcessing(entry: OutboxEntry): OutboxEntry { connectionProvider.withConnection { conn -> conn.prepareStatement(updateSql).use { stmt -> diff --git a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql new file mode 100644 index 0000000..a2f0692 --- /dev/null +++ b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql @@ -0,0 +1,6 @@ +--liquibase formatted sql +--changeset outbox:002 + +ALTER TABLE okapi_outbox + ADD INDEX idx_okapi_outbox_status_delivery_created_id (status, delivery_type, created_at, id), + ALGORITHM=INPLACE, LOCK=NONE; diff --git a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml index 390f883..571916b 100644 --- a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml +++ b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml @@ -4,4 +4,5 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd"> + diff --git a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt index 0ee29f5..f189ea1 100644 --- a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt +++ b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt @@ -4,7 +4,7 @@ import com.softwaremill.okapi.core.ConnectionProvider import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxId import com.softwaremill.okapi.core.OutboxStatus -import com.softwaremill.okapi.core.OutboxStore +import com.softwaremill.okapi.core.RouteAwareOutboxStore import java.sql.PreparedStatement import java.sql.ResultSet import java.sql.Timestamp @@ -15,7 +15,7 @@ import java.util.UUID /** PostgreSQL [OutboxStore] implementation using plain JDBC. */ class PostgresOutboxStore( private val connectionProvider: ConnectionProvider, -) : OutboxStore { +) : RouteAwareOutboxStore { override fun persist(entry: OutboxEntry): OutboxEntry { connectionProvider.withConnection { conn -> @@ -64,6 +64,27 @@ class PostgresOutboxStore( } } + override fun claimPending(deliveryType: String, limit: Int): List { + val sql = """ + SELECT * FROM okapi_outbox + WHERE status = ? AND delivery_type = ? + ORDER BY created_at ASC, id ASC + LIMIT ? + FOR UPDATE SKIP LOCKED + """.trimIndent() + + return connectionProvider.withConnection { conn -> + conn.prepareStatement(sql).use { stmt -> + stmt.setString(1, OutboxStatus.PENDING.name) + stmt.setString(2, deliveryType) + stmt.setInt(3, limit) + stmt.executeQuery().use { rs -> + generateSequence { if (rs.next()) rs.toOutboxEntry() else null }.toList() + } + } + } + } + override fun updateAfterProcessing(entry: OutboxEntry): OutboxEntry { connectionProvider.withConnection { conn -> conn.prepareStatement(updateSql).use { stmt -> diff --git a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql new file mode 100644 index 0000000..cbf5b67 --- /dev/null +++ b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql @@ -0,0 +1,5 @@ +--liquibase formatted sql +--changeset outbox:002 runInTransaction:false + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_okapi_outbox_status_delivery_created_id + ON okapi_outbox (status, delivery_type, created_at, id); diff --git a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/changelog.xml b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/changelog.xml index 390f883..571916b 100644 --- a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/changelog.xml +++ b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/changelog.xml @@ -4,4 +4,5 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd"> + diff --git a/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt b/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt index bb2a526..4a5ade8 100644 --- a/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt +++ b/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt @@ -10,6 +10,7 @@ import com.softwaremill.okapi.core.OutboxPurgerConfig import com.softwaremill.okapi.core.OutboxSchedulerConfig import com.softwaremill.okapi.core.OutboxStore import com.softwaremill.okapi.core.RetryPolicy +import com.softwaremill.okapi.core.RouteAwareOutboxStore import com.softwaremill.okapi.core.TransactionRunner import com.softwaremill.okapi.mysql.MysqlOutboxStore import com.softwaremill.okapi.postgres.PostgresOutboxStore @@ -193,6 +194,10 @@ class OutboxAutoConfiguration( "Okapi processor is enabled but no MessageDeliverer bean is registered. " + "Define a MessageDeliverer or set okapi.processor.enabled=false for publisher-only use." } + check(!processorEnabled || outboxStore is RouteAwareOutboxStore) { + "Okapi processor requires a RouteAwareOutboxStore. " + + "Implement route-aware claiming in the custom OutboxStore or set okapi.processor.enabled=false." + } return OutboxProcessor( store = outboxStore, entryProcessor = outboxEntryProcessor, diff --git a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OkapiSpringTestStubs.kt b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OkapiSpringTestStubs.kt index 855f7e6..e90793b 100644 --- a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OkapiSpringTestStubs.kt +++ b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OkapiSpringTestStubs.kt @@ -4,7 +4,7 @@ import com.softwaremill.okapi.core.DeliveryResult import com.softwaremill.okapi.core.MessageDeliverer import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxStatus -import com.softwaremill.okapi.core.OutboxStore +import com.softwaremill.okapi.core.RouteAwareOutboxStore import com.softwaremill.okapi.core.TransactionRunner import org.springframework.beans.factory.support.BeanDefinitionBuilder import org.springframework.context.support.GenericApplicationContext @@ -16,9 +16,10 @@ import java.time.Instant * `OutboxStore` interface change is a one-line edit, not a 7-file sweep. */ -internal fun stubStore() = object : OutboxStore { +internal fun stubStore() = object : RouteAwareOutboxStore { override fun persist(entry: OutboxEntry) = entry override fun claimPending(limit: Int) = emptyList() + override fun claimPending(deliveryType: String, limit: Int) = emptyList() override fun updateAfterProcessing(entry: OutboxEntry) = entry override fun removeDeliveredBefore(time: Instant, limit: Int) = 0 override fun findOldestCreatedAt(statuses: Set) = emptyMap() diff --git a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt index 62a3bf8..c75ab34 100644 --- a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt +++ b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OutboxProcessorAutoConfigurationTest.kt @@ -74,6 +74,20 @@ class OutboxProcessorAutoConfigurationTest : FunSpec({ } } + test("enabled processor rejects a store without route-aware claiming") { + ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(OutboxAutoConfiguration::class.java)) + .withBean(OutboxStore::class.java, { object : OutboxStore by stubStore() {} }) + .withBean(MessageDeliverer::class.java, { stubDeliverer() }) + .withBean(DataSource::class.java, { SimpleDriverDataSource() }) + .withBean(TransactionRunner::class.java, { noOpTransactionRunner() }) + .run { ctx -> + generateSequence(ctx.startupFailure.shouldNotBeNull()) { it.cause } + .mapNotNull { it.message } + .joinToString(" ") shouldContain "requires a RouteAwareOutboxStore" + } + } + test("publisher-only application starts without a deliverer") { ApplicationContextRunner() .withConfiguration(AutoConfigurations.of(OutboxAutoConfiguration::class.java)) From 8e3521302dd51447adb38eecb855ee4d92a6ccb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Kobyli=C5=84ski?= Date: Sat, 26 Sep 2026 12:42:45 +0200 Subject: [PATCH 4/5] perf: claim supported routes in at most two indexed queries --- README.md | 2 +- benchmarks/route-aware-claim/README.md | 20 ++++++------- .../okapi-116-my-queries.sql | 6 ++-- .../route-aware-claim/okapi-116-pg-setup.sql | 2 +- .../okapi/core/OutboxProcessor.kt | 27 ++++++++++------- .../okapi/core/RouteAwareOutboxStore.kt | 9 ++++-- .../okapi/core/OutboxProcessorTest.kt | 8 ++--- .../okapi/core/OutboxSchedulerTest.kt | 2 +- .../core/RouteAwareOutboxProcessorTest.kt | 23 +++++++++++---- .../test/store/OutboxStoreContractTests.kt | 29 +++++++++++++++++++ .../okapi/mysql/MysqlOutboxStore.kt | 17 +++++++---- .../okapi/postgres/PostgresOutboxStore.kt | 15 ++++++---- .../postgres/002__route_aware_claim_index.sql | 5 ++-- .../okapi/springboot/OkapiSpringTestStubs.kt | 2 +- 14 files changed, 114 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 8ee4061..1e252d0 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ Runnable, self-contained applications live in [okapi-examples](https://github.co - **Duplicate delivery is possible.** A crash between a successful delivery and the status update means the message may be sent again after restart. If processing a message more than once would cause unwanted effects, make the consumer idempotent — for example, deduplicate on a business key in the payload or a header set in the `DeliveryInfo`. okapi sends your payload and configured headers, but not the `OutboxId` returned by `publish()`; that identifier stays on the publisher side for correlation and logging. - **Best-effort ordering.** Rows are claimed by `created_at, id` within each delivery type. The processor rotates which type it checks first between batches. Parallel delivery and retries can also change arrival order. Strict delivery ordering across types or within a type is not guaranteed. -- **Different workers can handle different delivery types.** Each worker claims only its registered types. A custom `OutboxStore` used for processing must implement `RouteAwareOutboxStore`; otherwise processing fails before an unsafe claim. The older `claimPending(limit)` method remains available for callers that need it, but Okapi's processor does not use it. Apply the new index migration before starting updated workers if you manage the schema yourself. +- **Different workers can handle different delivery types.** Each worker claims only its registered types, using at most two claim queries per batch regardless of the number of types. A custom `OutboxStore` used for processing must implement `RouteAwareOutboxStore`; otherwise processing fails before an unsafe claim. The older `claimPending(limit)` method remains available for callers that need it, but Okapi's processor does not use it. Apply the new index migration before starting updated workers if you manage the schema yourself. - **Rolling upgrades need sequencing.** Workers running an older Okapi version can still claim rows of a newly introduced delivery type. Upgrade or stop all old processors before publishing that type. An upgrade does not automatically retry rows already marked `FAILED` by older workers. - **Failure classification is the transport's job.** Each deliverer decides what is retriable. HTTP: 5xx, 429, 408 and connection errors are retriable; other responses and TLS errors are permanent. Kafka: broker-side retriable exceptions are retried; authorization and configuration errors are not. - **Retry budget.** `okapi.processor.max-retries` (default 5) counts retries *after* the first attempt — six attempts in total before a row becomes `FAILED`. `FAILED` is terminal. Retriable messages become eligible again on the next processor poll; there is no per-message backoff. diff --git a/benchmarks/route-aware-claim/README.md b/benchmarks/route-aware-claim/README.md index 5fc2c06..46088d0 100644 --- a/benchmarks/route-aware-claim/README.md +++ b/benchmarks/route-aware-claim/README.md @@ -1,23 +1,23 @@ # Route-aware claiming on a large outbox -Local exploratory measurements with PostgreSQL 16 and MySQL 8.0 in Docker. The setup SQL files create a table with the Okapi columns and 1,000,000 `PENDING` rows. The oldest 900,000 use `delivery_type='unhandled'`; the newer 100,000 are split evenly across 100 `handled_00` through `handled_99` routes. Both the old `(status, created_at)` index and the new `(status, delivery_type, created_at, id)` index are present. +Exploratory local measurements with PostgreSQL 16 and MySQL 8.0 in Docker. The setup SQL creates 1,000,000 `PENDING` rows: 900,000 older rows have an unsupported `delivery_type`, while 100,000 newer rows are spread across 100 supported types. The legacy `(status, created_at)` index stays in place for the public unrestricted claim API. The new PostgreSQL index is partial on `(delivery_type, created_at, id) WHERE status = 'PENDING'`; MySQL adds `(status, delivery_type, created_at, id)`. -Run `okapi-116-pg-setup.sql` with `psql` or `okapi-116-my-setup.sql` with `mysql`, then run the matching `okapi-116-*-queries.sql` file. The query files compare one route, an `IN` query for 100 present routes, and an `IN` query for 100 absent routes. All use `FOR UPDATE SKIP LOCKED` and a batch limit of 100. The `okapi-116-*-client-100.sql` files send 100 separate empty claims through a SQL client. +Run the matching `*-setup.sql` and `*-queries.sql` files with `psql` or `mysql`. They compare one route with `IN` over 100 present or absent routes, all with `FOR UPDATE SKIP LOCKED` and limit 100. The `*-client-100.sql` files preserve an earlier alternative: 100 separate empty claims sent through one local SQL client process. The processor now uses **at most two claims per batch**: one for a rotating starting type, then one `IN` claim for all remaining types when the first claim did not fill the batch. | Case | PostgreSQL observed execution | MySQL observed execution | | --- | ---: | ---: | -| One present route, first 100 rows | 0.12 ms | 0.49 ms | -| `IN` with 100 absent routes, ordered by type and creation time | 165 ms | 0.21 ms | +| One present route, first 100 rows | ~0.15 ms | ~0.26–0.49 ms | +| `IN` with 100 absent routes, ordered by type and creation time | ~0.16 ms | ~0.21 ms | | 100 separate empty route claims, server-side loop | 1.57 ms | 2.04–2.37 ms | | 100 separate empty route claims through one local SQL client process | 24 ms | 15 ms | -The PostgreSQL plan for the absent `IN` case scanned all 1,000,000 rows and removed them by filter, despite the new index. The one-route plan on both databases used the new composite index and stopped at the limit. This is why the processor claims one route at a time and stops once its batch is full. At 100 empty routes it still performs 100 small queries per poll; this cost grows with the number of configured routes. +Both current `IN` plans use the new route index, including PostgreSQL with a generic prepared plan. The PostgreSQL query must contain literal `status = 'PENDING'` so a generic plan can use its partial index. A preliminary full PostgreSQL index `(status, delivery_type, created_at, id)` produced a bad plan for the 100-absent-type query: it scanned 1,000,000 rows in ~165 ms. The partial index avoids that observed plan. The MySQL implementation also checks `BINARY delivery_type` to preserve exact type matching under case-insensitive database collations; the primary `IN` predicate still drives the index range scan. -The `okapi-116-*-write.sql` files insert 20,000 rows and update about 20,000 statuses inside rolled-back transactions. One run with the new index and one after dropping only that index gave: +The extra index costs writes. In five alternating series of 10,000 committed inserts and status updates against million-row tables, median times with / without the additional route index were: -| Operation | PostgreSQL with / without new index | MySQL with / without new index | +| Operation | PostgreSQL partial index | MySQL full index | | --- | ---: | ---: | -| Insert 20,000 | 125 / 81 ms | 177 / 141 ms | -| Update about 20,000 statuses | 158 / 135 ms | 363 / 304 ms | +| Insert 10,000 | 57.46 / 39.03 ms (1.47×) | 94.95 / 80.63 ms (1.18×) | +| Update 10,000 statuses | 88.36 / 82.56 ms (1.07×) | 204.30 / 158.82 ms (1.29×) | -These write figures are **single local runs**, sensitive to cache state and test order. They show a real write-cost tradeoff, not a stable throughput estimate. The read measurements are also exploratory; the query plan and scanned-row counts are the main evidence. Integration tests separately cover route isolation, concurrent processors, migrations, and delivery behavior on both databases. +The `*-write.sql` files provide a small, rollback-based write probe; the table above is from repeated committed runs on comparable million-row tables, so its figures are not directly reproduced by those scripts. Results depend on cache, hardware, row distribution and workload. The plans and scanned-row counts matter more than these timings. Removing the legacy index made unrestricted `claimPending(limit)` very slow in this dataset, so both indexes remain. Integration tests separately cover migrations, route isolation, exact type matching, multiple routes and concurrent workers on both databases. diff --git a/benchmarks/route-aware-claim/okapi-116-my-queries.sql b/benchmarks/route-aware-claim/okapi-116-my-queries.sql index d51c030..5f38c8d 100644 --- a/benchmarks/route-aware-claim/okapi-116-my-queries.sql +++ b/benchmarks/route-aware-claim/okapi-116-my-queries.sql @@ -1,6 +1,6 @@ USE okapi_bench; START TRANSACTION; -EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='handled_00' ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; -EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type IN ('handled_00', 'handled_01', 'handled_02', 'handled_03', 'handled_04', 'handled_05', 'handled_06', 'handled_07', 'handled_08', 'handled_09', 'handled_10', 'handled_11', 'handled_12', 'handled_13', 'handled_14', 'handled_15', 'handled_16', 'handled_17', 'handled_18', 'handled_19', 'handled_20', 'handled_21', 'handled_22', 'handled_23', 'handled_24', 'handled_25', 'handled_26', 'handled_27', 'handled_28', 'handled_29', 'handled_30', 'handled_31', 'handled_32', 'handled_33', 'handled_34', 'handled_35', 'handled_36', 'handled_37', 'handled_38', 'handled_39', 'handled_40', 'handled_41', 'handled_42', 'handled_43', 'handled_44', 'handled_45', 'handled_46', 'handled_47', 'handled_48', 'handled_49', 'handled_50', 'handled_51', 'handled_52', 'handled_53', 'handled_54', 'handled_55', 'handled_56', 'handled_57', 'handled_58', 'handled_59', 'handled_60', 'handled_61', 'handled_62', 'handled_63', 'handled_64', 'handled_65', 'handled_66', 'handled_67', 'handled_68', 'handled_69', 'handled_70', 'handled_71', 'handled_72', 'handled_73', 'handled_74', 'handled_75', 'handled_76', 'handled_77', 'handled_78', 'handled_79', 'handled_80', 'handled_81', 'handled_82', 'handled_83', 'handled_84', 'handled_85', 'handled_86', 'handled_87', 'handled_88', 'handled_89', 'handled_90', 'handled_91', 'handled_92', 'handled_93', 'handled_94', 'handled_95', 'handled_96', 'handled_97', 'handled_98', 'handled_99') ORDER BY delivery_type,created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; -EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type IN ('absent_00', 'absent_01', 'absent_02', 'absent_03', 'absent_04', 'absent_05', 'absent_06', 'absent_07', 'absent_08', 'absent_09', 'absent_10', 'absent_11', 'absent_12', 'absent_13', 'absent_14', 'absent_15', 'absent_16', 'absent_17', 'absent_18', 'absent_19', 'absent_20', 'absent_21', 'absent_22', 'absent_23', 'absent_24', 'absent_25', 'absent_26', 'absent_27', 'absent_28', 'absent_29', 'absent_30', 'absent_31', 'absent_32', 'absent_33', 'absent_34', 'absent_35', 'absent_36', 'absent_37', 'absent_38', 'absent_39', 'absent_40', 'absent_41', 'absent_42', 'absent_43', 'absent_44', 'absent_45', 'absent_46', 'absent_47', 'absent_48', 'absent_49', 'absent_50', 'absent_51', 'absent_52', 'absent_53', 'absent_54', 'absent_55', 'absent_56', 'absent_57', 'absent_58', 'absent_59', 'absent_60', 'absent_61', 'absent_62', 'absent_63', 'absent_64', 'absent_65', 'absent_66', 'absent_67', 'absent_68', 'absent_69', 'absent_70', 'absent_71', 'absent_72', 'absent_73', 'absent_74', 'absent_75', 'absent_76', 'absent_77', 'absent_78', 'absent_79', 'absent_80', 'absent_81', 'absent_82', 'absent_83', 'absent_84', 'absent_85', 'absent_86', 'absent_87', 'absent_88', 'absent_89', 'absent_90', 'absent_91', 'absent_92', 'absent_93', 'absent_94', 'absent_95', 'absent_96', 'absent_97', 'absent_98', 'absent_99') ORDER BY delivery_type,created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type='handled_00' AND BINARY delivery_type IN (BINARY 'handled_00') ORDER BY created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type IN ('handled_00', 'handled_01', 'handled_02', 'handled_03', 'handled_04', 'handled_05', 'handled_06', 'handled_07', 'handled_08', 'handled_09', 'handled_10', 'handled_11', 'handled_12', 'handled_13', 'handled_14', 'handled_15', 'handled_16', 'handled_17', 'handled_18', 'handled_19', 'handled_20', 'handled_21', 'handled_22', 'handled_23', 'handled_24', 'handled_25', 'handled_26', 'handled_27', 'handled_28', 'handled_29', 'handled_30', 'handled_31', 'handled_32', 'handled_33', 'handled_34', 'handled_35', 'handled_36', 'handled_37', 'handled_38', 'handled_39', 'handled_40', 'handled_41', 'handled_42', 'handled_43', 'handled_44', 'handled_45', 'handled_46', 'handled_47', 'handled_48', 'handled_49', 'handled_50', 'handled_51', 'handled_52', 'handled_53', 'handled_54', 'handled_55', 'handled_56', 'handled_57', 'handled_58', 'handled_59', 'handled_60', 'handled_61', 'handled_62', 'handled_63', 'handled_64', 'handled_65', 'handled_66', 'handled_67', 'handled_68', 'handled_69', 'handled_70', 'handled_71', 'handled_72', 'handled_73', 'handled_74', 'handled_75', 'handled_76', 'handled_77', 'handled_78', 'handled_79', 'handled_80', 'handled_81', 'handled_82', 'handled_83', 'handled_84', 'handled_85', 'handled_86', 'handled_87', 'handled_88', 'handled_89', 'handled_90', 'handled_91', 'handled_92', 'handled_93', 'handled_94', 'handled_95', 'handled_96', 'handled_97', 'handled_98', 'handled_99') AND BINARY delivery_type IN (BINARY 'handled_00', BINARY 'handled_01', BINARY 'handled_02', BINARY 'handled_03', BINARY 'handled_04', BINARY 'handled_05', BINARY 'handled_06', BINARY 'handled_07', BINARY 'handled_08', BINARY 'handled_09', BINARY 'handled_10', BINARY 'handled_11', BINARY 'handled_12', BINARY 'handled_13', BINARY 'handled_14', BINARY 'handled_15', BINARY 'handled_16', BINARY 'handled_17', BINARY 'handled_18', BINARY 'handled_19', BINARY 'handled_20', BINARY 'handled_21', BINARY 'handled_22', BINARY 'handled_23', BINARY 'handled_24', BINARY 'handled_25', BINARY 'handled_26', BINARY 'handled_27', BINARY 'handled_28', BINARY 'handled_29', BINARY 'handled_30', BINARY 'handled_31', BINARY 'handled_32', BINARY 'handled_33', BINARY 'handled_34', BINARY 'handled_35', BINARY 'handled_36', BINARY 'handled_37', BINARY 'handled_38', BINARY 'handled_39', BINARY 'handled_40', BINARY 'handled_41', BINARY 'handled_42', BINARY 'handled_43', BINARY 'handled_44', BINARY 'handled_45', BINARY 'handled_46', BINARY 'handled_47', BINARY 'handled_48', BINARY 'handled_49', BINARY 'handled_50', BINARY 'handled_51', BINARY 'handled_52', BINARY 'handled_53', BINARY 'handled_54', BINARY 'handled_55', BINARY 'handled_56', BINARY 'handled_57', BINARY 'handled_58', BINARY 'handled_59', BINARY 'handled_60', BINARY 'handled_61', BINARY 'handled_62', BINARY 'handled_63', BINARY 'handled_64', BINARY 'handled_65', BINARY 'handled_66', BINARY 'handled_67', BINARY 'handled_68', BINARY 'handled_69', BINARY 'handled_70', BINARY 'handled_71', BINARY 'handled_72', BINARY 'handled_73', BINARY 'handled_74', BINARY 'handled_75', BINARY 'handled_76', BINARY 'handled_77', BINARY 'handled_78', BINARY 'handled_79', BINARY 'handled_80', BINARY 'handled_81', BINARY 'handled_82', BINARY 'handled_83', BINARY 'handled_84', BINARY 'handled_85', BINARY 'handled_86', BINARY 'handled_87', BINARY 'handled_88', BINARY 'handled_89', BINARY 'handled_90', BINARY 'handled_91', BINARY 'handled_92', BINARY 'handled_93', BINARY 'handled_94', BINARY 'handled_95', BINARY 'handled_96', BINARY 'handled_97', BINARY 'handled_98', BINARY 'handled_99') ORDER BY delivery_type,created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; +EXPLAIN ANALYZE SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status='PENDING' AND delivery_type IN ('absent_00', 'absent_01', 'absent_02', 'absent_03', 'absent_04', 'absent_05', 'absent_06', 'absent_07', 'absent_08', 'absent_09', 'absent_10', 'absent_11', 'absent_12', 'absent_13', 'absent_14', 'absent_15', 'absent_16', 'absent_17', 'absent_18', 'absent_19', 'absent_20', 'absent_21', 'absent_22', 'absent_23', 'absent_24', 'absent_25', 'absent_26', 'absent_27', 'absent_28', 'absent_29', 'absent_30', 'absent_31', 'absent_32', 'absent_33', 'absent_34', 'absent_35', 'absent_36', 'absent_37', 'absent_38', 'absent_39', 'absent_40', 'absent_41', 'absent_42', 'absent_43', 'absent_44', 'absent_45', 'absent_46', 'absent_47', 'absent_48', 'absent_49', 'absent_50', 'absent_51', 'absent_52', 'absent_53', 'absent_54', 'absent_55', 'absent_56', 'absent_57', 'absent_58', 'absent_59', 'absent_60', 'absent_61', 'absent_62', 'absent_63', 'absent_64', 'absent_65', 'absent_66', 'absent_67', 'absent_68', 'absent_69', 'absent_70', 'absent_71', 'absent_72', 'absent_73', 'absent_74', 'absent_75', 'absent_76', 'absent_77', 'absent_78', 'absent_79', 'absent_80', 'absent_81', 'absent_82', 'absent_83', 'absent_84', 'absent_85', 'absent_86', 'absent_87', 'absent_88', 'absent_89', 'absent_90', 'absent_91', 'absent_92', 'absent_93', 'absent_94', 'absent_95', 'absent_96', 'absent_97', 'absent_98', 'absent_99') AND BINARY delivery_type IN (BINARY 'absent_00', BINARY 'absent_01', BINARY 'absent_02', BINARY 'absent_03', BINARY 'absent_04', BINARY 'absent_05', BINARY 'absent_06', BINARY 'absent_07', BINARY 'absent_08', BINARY 'absent_09', BINARY 'absent_10', BINARY 'absent_11', BINARY 'absent_12', BINARY 'absent_13', BINARY 'absent_14', BINARY 'absent_15', BINARY 'absent_16', BINARY 'absent_17', BINARY 'absent_18', BINARY 'absent_19', BINARY 'absent_20', BINARY 'absent_21', BINARY 'absent_22', BINARY 'absent_23', BINARY 'absent_24', BINARY 'absent_25', BINARY 'absent_26', BINARY 'absent_27', BINARY 'absent_28', BINARY 'absent_29', BINARY 'absent_30', BINARY 'absent_31', BINARY 'absent_32', BINARY 'absent_33', BINARY 'absent_34', BINARY 'absent_35', BINARY 'absent_36', BINARY 'absent_37', BINARY 'absent_38', BINARY 'absent_39', BINARY 'absent_40', BINARY 'absent_41', BINARY 'absent_42', BINARY 'absent_43', BINARY 'absent_44', BINARY 'absent_45', BINARY 'absent_46', BINARY 'absent_47', BINARY 'absent_48', BINARY 'absent_49', BINARY 'absent_50', BINARY 'absent_51', BINARY 'absent_52', BINARY 'absent_53', BINARY 'absent_54', BINARY 'absent_55', BINARY 'absent_56', BINARY 'absent_57', BINARY 'absent_58', BINARY 'absent_59', BINARY 'absent_60', BINARY 'absent_61', BINARY 'absent_62', BINARY 'absent_63', BINARY 'absent_64', BINARY 'absent_65', BINARY 'absent_66', BINARY 'absent_67', BINARY 'absent_68', BINARY 'absent_69', BINARY 'absent_70', BINARY 'absent_71', BINARY 'absent_72', BINARY 'absent_73', BINARY 'absent_74', BINARY 'absent_75', BINARY 'absent_76', BINARY 'absent_77', BINARY 'absent_78', BINARY 'absent_79', BINARY 'absent_80', BINARY 'absent_81', BINARY 'absent_82', BINARY 'absent_83', BINARY 'absent_84', BINARY 'absent_85', BINARY 'absent_86', BINARY 'absent_87', BINARY 'absent_88', BINARY 'absent_89', BINARY 'absent_90', BINARY 'absent_91', BINARY 'absent_92', BINARY 'absent_93', BINARY 'absent_94', BINARY 'absent_95', BINARY 'absent_96', BINARY 'absent_97', BINARY 'absent_98', BINARY 'absent_99') ORDER BY delivery_type,created_at,id LIMIT 100 FOR UPDATE SKIP LOCKED; ROLLBACK; diff --git a/benchmarks/route-aware-claim/okapi-116-pg-setup.sql b/benchmarks/route-aware-claim/okapi-116-pg-setup.sql index 4ef042a..3101849 100644 --- a/benchmarks/route-aware-claim/okapi-116-pg-setup.sql +++ b/benchmarks/route-aware-claim/okapi-116-pg-setup.sql @@ -18,5 +18,5 @@ SELECT md5(n::text)::uuid, 'event', '{}', timestamp '2024-01-01', 0, NULL, NULL, '{}'::jsonb FROM generate_series(1, 1000000) AS n; CREATE INDEX idx_okapi_outbox_status_created_at ON okapi_outbox (status, created_at); -CREATE INDEX idx_okapi_outbox_status_delivery_created_id ON okapi_outbox (status, delivery_type, created_at, id); +CREATE INDEX idx_okapi_outbox_pending_delivery_created_id ON okapi_outbox (delivery_type, created_at, id) WHERE status = 'PENDING'; ANALYZE okapi_outbox; diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt index a57ae78..2f92540 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt @@ -38,17 +38,24 @@ class OutboxProcessor @JvmOverloads constructor( val routeAwareStore = store as? RouteAwareOutboxStore ?: error("OutboxProcessor requires a RouteAwareOutboxStore to avoid claiming unsupported delivery types") check(deliveryTypes.isNotEmpty()) { "OutboxProcessor requires at least one delivery type" } - val claimed = buildList { - val start = Math.floorMod(nextStartingType.getAndIncrement(), deliveryTypes.size) - for (offset in deliveryTypes.indices) { - val remaining = limit - size - if (remaining <= 0) break - val deliveryType = deliveryTypes[(start + offset) % deliveryTypes.size] - val entries = routeAwareStore.claimPending(deliveryType, remaining) - check(entries.size <= remaining && entries.all { it.deliveryType == deliveryType }) { - "RouteAwareOutboxStore returned entries outside the requested type or limit" + val claimed = if (limit <= 0) { + emptyList() + } else { + val startingType = deliveryTypes[Math.floorMod(nextStartingType.getAndIncrement(), deliveryTypes.size)] + val first = routeAwareStore.claimPending(startingType, limit) + check(first.size <= limit && first.all { it.deliveryType == startingType }) { + "RouteAwareOutboxStore returned entries outside the requested type or limit" + } + val remainingTypes = deliveryTypes.filterTo(LinkedHashSet()) { it != startingType } + if (first.size == limit || remainingTypes.isEmpty()) { + first + } else { + val remaining = limit - first.size + val other = routeAwareStore.claimPending(remainingTypes, remaining) + check(other.size <= remaining && other.all { it.deliveryType in remainingTypes }) { + "RouteAwareOutboxStore returned entries outside the requested types or limit" } - addAll(entries) + first + other } } if (claimed.isEmpty()) { diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxStore.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxStore.kt index 08483b4..f036c82 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxStore.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxStore.kt @@ -1,11 +1,14 @@ package com.softwaremill.okapi.core /** - * Claims only entries for a delivery type handled by the current worker. + * Claims only entries for delivery types handled by the current worker. * The returned rows must be locked until the processing transaction completes. - * Implementations should return at most the requested number of rows ordered by creation time and ID. + * Implementations should return at most the requested number of rows ordered by delivery type, + * creation time and ID. * The processor rejects results with another [OutboxEntry.deliveryType] or more than requested. */ interface RouteAwareOutboxStore : OutboxStore { - fun claimPending(deliveryType: String, limit: Int): List + fun claimPending(deliveryTypes: Set, limit: Int): List + + fun claimPending(deliveryType: String, limit: Int): List = claimPending(setOf(deliveryType), limit) } diff --git a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxProcessorTest.kt b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxProcessorTest.kt index effda8f..bc17778 100644 --- a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxProcessorTest.kt +++ b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxProcessorTest.kt @@ -35,8 +35,8 @@ class OutboxProcessorTest : override fun claimPending(limit: Int): List = pendingEntries.take(limit) - override fun claimPending(deliveryType: String, limit: Int): List = - pendingEntries.filter { it.deliveryType == deliveryType }.take(limit) + override fun claimPending(deliveryTypes: Set, limit: Int): List = + pendingEntries.filter { it.deliveryType in deliveryTypes }.take(limit) override fun updateAfterProcessing(entry: OutboxEntry): OutboxEntry = entry.also { processedEntries += it } @@ -340,8 +340,8 @@ class OutboxProcessorTest : val batchAwareStore = object : RouteAwareOutboxStore { override fun persist(entry: OutboxEntry) = entry override fun claimPending(limit: Int): List = pendingEntries.take(limit) - override fun claimPending(deliveryType: String, limit: Int): List = - pendingEntries.filter { it.deliveryType == deliveryType }.take(limit) + override fun claimPending(deliveryTypes: Set, limit: Int): List = + pendingEntries.filter { it.deliveryType in deliveryTypes }.take(limit) override fun updateAfterProcessing(entry: OutboxEntry): OutboxEntry = entry.also { individualCalls += it } override fun removeDeliveredBefore(time: Instant, limit: Int) = 0 override fun findOldestCreatedAt(statuses: Set) = emptyMap() diff --git a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxSchedulerTest.kt b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxSchedulerTest.kt index 7ee10d9..e250ead 100644 --- a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxSchedulerTest.kt +++ b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxSchedulerTest.kt @@ -692,7 +692,7 @@ private fun stubProcessor(onProcessNext: (Int) -> Unit): OutboxProcessor { onProcessNext(limit) return emptyList() } - override fun claimPending(deliveryType: String, limit: Int): List { + override fun claimPending(deliveryTypes: Set, limit: Int): List { onProcessNext(limit) return emptyList() } diff --git a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxProcessorTest.kt b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxProcessorTest.kt index ebfb185..c418b65 100644 --- a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxProcessorTest.kt +++ b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/RouteAwareOutboxProcessorTest.kt @@ -22,14 +22,14 @@ private fun routeDeliverer(type: String) = object : MessageDeliverer { } private class RoutingStore(var pending: List) : RouteAwareOutboxStore { - val queriedTypes = mutableListOf() + val queriedTypes = mutableListOf>() val processed = mutableListOf() override fun persist(entry: OutboxEntry) = entry override fun claimPending(limit: Int): List = error("Unsafe claimPending(limit) must not be used") - override fun claimPending(deliveryType: String, limit: Int): List { - queriedTypes += deliveryType - val claimed = pending.filter { it.deliveryType == deliveryType }.take(limit) + override fun claimPending(deliveryTypes: Set, limit: Int): List { + queriedTypes += deliveryTypes + val claimed = pending.filter { it.deliveryType in deliveryTypes }.take(limit) pending = pending - claimed.toSet() return claimed } @@ -58,7 +58,7 @@ class RouteAwareOutboxProcessorTest : FunSpec({ processor.processNext(10) shouldBe 1 store.pending shouldContainExactly listOf(unsupported) store.processed.single().status shouldBe OutboxStatus.DELIVERED - store.queriedTypes shouldContainExactly listOf("kafka") + store.queriedTypes shouldContainExactly listOf(setOf("kafka")) } test("starting route rotates and claims at most the batch limit") { @@ -68,10 +68,21 @@ class RouteAwareOutboxProcessorTest : FunSpec({ processor.processNext(1) shouldBe 1 processor.processNext(1) shouldBe 1 - store.queriedTypes shouldContainExactly listOf("kafka", "http") + store.queriedTypes shouldContainExactly listOf(setOf("kafka"), setOf("http")) store.processed.map { it.deliveryType } shouldContainExactly listOf("kafka", "http") } + test("an empty poll with many routes uses at most two store calls") { + val store = RoutingStore(emptyList()) + val deliverer = CompositeMessageDeliverer((0 until 100).map { routeDeliverer("route_$it") }) + val processor = OutboxProcessor(store, OutboxEntryProcessor(deliverer, RetryPolicy(1), Clock.systemUTC())) + + processor.processNext(10) shouldBe 0 + store.queriedTypes.size shouldBe 2 + store.queriedTypes.first().size shouldBe 1 + store.queriedTypes.last().size shouldBe 99 + } + test("processing with a store lacking route-aware claims fails before legacy claim") { val store = object : OutboxStore by RoutingStore(emptyList()) { override fun claimPending(limit: Int): List = error("legacy claim was called") diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/OutboxStoreContractTests.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/OutboxStoreContractTests.kt index bee7d43..fd5a67a 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/OutboxStoreContractTests.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/OutboxStoreContractTests.kt @@ -95,6 +95,35 @@ fun FunSpec.outboxStoreContractTests( jdbc.withTransaction { routed.claimPending("http", 10) }.map { it.outboxId } shouldBe listOf(unsupported.outboxId) } + test("[$dbName] route-aware claim matches delivery types exactly") { + val upper = createTestEntry(deliveryInfo = StubDeliveryInfo("KAFKA")) + val lower = createTestEntry(deliveryInfo = StubDeliveryInfo("kafka")) + jdbc.withTransaction { + store.persist(upper) + store.persist(lower) + } + + val routed = store as RouteAwareOutboxStore + jdbc.withTransaction { routed.claimPending("kafka", 10) }.map { it.outboxId } shouldBe listOf(lower.outboxId) + jdbc.withTransaction { routed.claimPending("KAFKA", 10) }.map { it.outboxId } shouldBe listOf(upper.outboxId) + } + + test("[$dbName] route-aware claim selects a limited batch across several types") { + val ignored = createTestEntry(deliveryInfo = StubDeliveryInfo("ignored")) + val kafka = createTestEntry(deliveryInfo = StubDeliveryInfo("kafka")) + val http = createTestEntry(deliveryInfo = StubDeliveryInfo("http")) + jdbc.withTransaction { + store.persist(ignored) + store.persist(kafka) + store.persist(http) + } + + val routed = store as RouteAwareOutboxStore + val claimed = jdbc.withTransaction { routed.claimPending(setOf("kafka", "http"), 2) } + + claimed.map { it.outboxId } shouldBe listOf(http.outboxId, kafka.outboxId) + } + test("[$dbName] claimPending returns entries ordered by created_at ASC") { val t1 = Instant.parse("2024-01-01T00:00:00Z") val t2 = Instant.parse("2024-01-02T00:00:00Z") diff --git a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt index 8803ade..b9a90c3 100644 --- a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt +++ b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt @@ -68,13 +68,19 @@ class MysqlOutboxStore( } } - override fun claimPending(deliveryType: String, limit: Int): List { + override fun claimPending(deliveryTypes: Set, limit: Int): List { + if (deliveryTypes.isEmpty() || limit <= 0) return emptyList() + val routes = deliveryTypes.sorted() + val placeholders = routes.joinToString(",") { "?" } + val binaryPlaceholders = routes.joinToString(",") { "BINARY ?" } // A matching index avoids scanning and locking rows for other delivery types. + // The binary predicate excludes case-insensitive collation matches for another route. val sql = """ SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) - WHERE status = ? AND delivery_type = ? - ORDER BY created_at ASC, id ASC + WHERE status = ? AND delivery_type IN ($placeholders) + AND BINARY delivery_type IN ($binaryPlaceholders) + ORDER BY delivery_type ASC, created_at ASC, id ASC LIMIT ? FOR UPDATE SKIP LOCKED """.trimIndent() @@ -82,8 +88,9 @@ class MysqlOutboxStore( return connectionProvider.withConnection { conn -> conn.prepareStatement(sql).use { stmt -> stmt.setString(1, OutboxStatus.PENDING.name) - stmt.setString(2, deliveryType) - stmt.setInt(3, limit) + routes.forEachIndexed { index, route -> stmt.setString(index + 2, route) } + routes.forEachIndexed { index, route -> stmt.setString(index + routes.size + 2, route) } + stmt.setInt(routes.size * 2 + 2, limit) stmt.executeQuery().use { rs -> generateSequence { if (rs.next()) rs.toOutboxEntry() else null }.toList() } diff --git a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt index f189ea1..1cfa0d5 100644 --- a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt +++ b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt @@ -64,20 +64,23 @@ class PostgresOutboxStore( } } - override fun claimPending(deliveryType: String, limit: Int): List { + override fun claimPending(deliveryTypes: Set, limit: Int): List { + if (deliveryTypes.isEmpty() || limit <= 0) return emptyList() + val routes = deliveryTypes.sorted() + val placeholders = routes.joinToString(",") { "?" } + // Keep PENDING literal so a generic prepared plan can use the partial index. val sql = """ SELECT * FROM okapi_outbox - WHERE status = ? AND delivery_type = ? - ORDER BY created_at ASC, id ASC + WHERE status = 'PENDING' AND delivery_type IN ($placeholders) + ORDER BY delivery_type ASC, created_at ASC, id ASC LIMIT ? FOR UPDATE SKIP LOCKED """.trimIndent() return connectionProvider.withConnection { conn -> conn.prepareStatement(sql).use { stmt -> - stmt.setString(1, OutboxStatus.PENDING.name) - stmt.setString(2, deliveryType) - stmt.setInt(3, limit) + routes.forEachIndexed { index, route -> stmt.setString(index + 1, route) } + stmt.setInt(routes.size + 1, limit) stmt.executeQuery().use { rs -> generateSequence { if (rs.next()) rs.toOutboxEntry() else null }.toList() } diff --git a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql index cbf5b67..1dc8a46 100644 --- a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql +++ b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql @@ -1,5 +1,6 @@ --liquibase formatted sql --changeset outbox:002 runInTransaction:false -CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_okapi_outbox_status_delivery_created_id - ON okapi_outbox (status, delivery_type, created_at, id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_okapi_outbox_pending_delivery_created_id + ON okapi_outbox (delivery_type, created_at, id) + WHERE status = 'PENDING'; diff --git a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OkapiSpringTestStubs.kt b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OkapiSpringTestStubs.kt index e90793b..a8c7bc6 100644 --- a/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OkapiSpringTestStubs.kt +++ b/okapi-spring-boot/src/test/kotlin/com/softwaremill/okapi/springboot/OkapiSpringTestStubs.kt @@ -19,7 +19,7 @@ import java.time.Instant internal fun stubStore() = object : RouteAwareOutboxStore { override fun persist(entry: OutboxEntry) = entry override fun claimPending(limit: Int) = emptyList() - override fun claimPending(deliveryType: String, limit: Int) = emptyList() + override fun claimPending(deliveryTypes: Set, limit: Int) = emptyList() override fun updateAfterProcessing(entry: OutboxEntry) = entry override fun removeDeliveredBefore(time: Instant, limit: Int) = 0 override fun findOldestCreatedAt(statuses: Set) = emptyMap() From a69a9ae18a58ac702832cd7c81ee55cfd5b638c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Wokacz?= Date: Tue, 3 Nov 2026 14:09:14 +0100 Subject: [PATCH 5/5] fix(route-aware-claim): address review findings - Postgres 002: drop a leftover INVALID index before CREATE INDEX CONCURRENTLY IF NOT EXISTS, so a retried changeset rebuilds it. - MySQL claim: CAST(... AS BINARY) instead of the BINARY operator, deprecated since 8.0.27 (warning 1287 on every claim). Test asserts the claim raises no server warnings. - Name the store class and mention wrappers/proxies in the RouteAwareOutboxStore errors (processor and Spring autoconfig). - README: cross-type claim order favours alphabetically earlier types for leftover capacity; CREATE INDEX CONCURRENTLY waits for all older transactions in the database. - CHANGELOG: BREAKING entries for RouteAwareOutboxStore, process() throwing on unsupported types, migration 002; Fixed entry for #116. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 29 +++++++++++++++++++ README.md | 4 +-- .../okapi/core/OutboxProcessor.kt | 7 ++++- .../okapi/test/store/MysqlOutboxStoreTest.kt | 17 +++++++++++ .../okapi/mysql/MysqlOutboxStore.kt | 5 ++-- .../postgres/002__route_aware_claim_index.sql | 4 +++ .../springboot/OutboxAutoConfiguration.kt | 6 ++-- 7 files changed, 65 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fdd6ed..58d274a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,35 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * `OutboxStore`/`OutboxPublisher` operation routes through `withConnection`. Construct one instance per `Database`, matching `ExposedTransactionRunner` and `ExposedTransactionContextValidator`. ([#96](https://github.com/softwaremill/okapi/issues/96)) +- **`OutboxProcessor` requires a `RouteAwareOutboxStore`.** The processor now claims only the + delivery types its `OutboxEntryProcessor` handles, through the new + `RouteAwareOutboxStore.claimPending(deliveryTypes, limit)`; `PostgresOutboxStore` and + `MysqlOutboxStore` implement it. A custom `OutboxStore`, or a hand-written wrapper around a + built-in store, must implement `RouteAwareOutboxStore` too: otherwise `processNext` throws, and + Spring autoconfiguration fails at startup while the processor is enabled. `claimPending(limit)` + stays on `OutboxStore` for direct callers. ([#120](https://github.com/softwaremill/okapi/pull/120)) +- **`OutboxEntryProcessor.process`/`processBatch` throw `IllegalStateException` for an entry whose + delivery type has no registered deliverer.** Previously `CompositeMessageDeliverer` returned a + `PermanentFailure` and the entry was marked `FAILED`. Inside `OutboxProcessor` the exception rolls + the transaction back and the row stays `PENDING`; direct callers must handle the exception. + ([#120](https://github.com/softwaremill/okapi/pull/120)) +- **New migration `002__route_aware_claim_index`** adds a `(status, delivery_type, created_at, id)` + index on MySQL and a partial `(delivery_type, created_at, id) WHERE status = 'PENDING'` index on + PostgreSQL. The bundled Liquibase changelogs apply it automatically. If you manage the schema + yourself, apply it before starting updated workers: the MySQL claim uses `FORCE INDEX` and fails + without it, and the PostgreSQL script must run outside a transaction (`CREATE INDEX CONCURRENTLY`). + ([#120](https://github.com/softwaremill/okapi/pull/120)) + +### Fixed + +- **A worker no longer marks rows it cannot deliver as `FAILED`.** Previously every worker claimed + all `PENDING` rows, so a worker without a deliverer for a row's type turned it into a permanent + failure. Each worker now claims only its registered delivery types, in at most two queries per + batch. Workers on an older version still claim every type, so publish a newly introduced type only + after all old processors are upgraded or stopped. Rows already marked `FAILED` are not retried + automatically. Cross-type claim order is no longer globally oldest first: the processor rotates + the starting type, and any leftover batch capacity goes to the other types alphabetically. + ([#116](https://github.com/softwaremill/okapi/issues/116)) ## [1.0.0] — 2026-07-28 diff --git a/README.md b/README.md index bcf28b7..ea1d04b 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Runnable, self-contained applications live in [okapi-examples](https://github.co ### Guarantees and limits - **Duplicate delivery is possible.** A crash between a successful delivery and the status update means the message may be sent again after restart. If processing a message more than once would cause unwanted effects, make the consumer idempotent. okapi sends your payload, your configured headers, and an `x-outbox-id` header to deduplicate on — see [Deduplicating on `x-outbox-id`](#deduplicating-on-x-outbox-id). -- **Best-effort ordering.** Rows are claimed by `created_at, id` within each delivery type. The processor rotates which type it checks first between batches. Parallel delivery and retries can also change arrival order. Strict delivery ordering across types or within a type is not guaranteed. +- **Best-effort ordering.** Rows are claimed by `created_at, id` within each delivery type. The processor rotates which type it checks first between batches; when that type has fewer rows than the batch size, the remaining capacity goes to the other types in alphabetical order, so under mixed backlog alphabetically earlier types get more of it. Every type is still claimed first at least once every N batches, where N is the number of delivery types the worker handles. Across types, rows are no longer claimed globally oldest first. Parallel delivery and retries can also change arrival order. Strict delivery ordering across types or within a type is not guaranteed. - **Different workers can handle different delivery types.** Each worker claims only its registered types, using at most two claim queries per batch regardless of the number of types. A custom `OutboxStore` used for processing must implement `RouteAwareOutboxStore`; otherwise processing fails before an unsafe claim. The older `claimPending(limit)` method remains available for callers that need it, but Okapi's processor does not use it. Apply the new index migration before starting updated workers if you manage the schema yourself. - **Rolling upgrades need sequencing.** Workers running an older Okapi version can still claim rows of a newly introduced delivery type. Upgrade or stop all old processors before publishing that type. An upgrade does not automatically retry rows already marked `FAILED` by older workers. - **Failure classification is the transport's job.** Each deliverer decides what is retriable. HTTP: 5xx, 429, 408 and connection errors are retriable; other responses and TLS errors are permanent. Kafka: broker-side retriable exceptions are retried; authorization and configuration errors are not. @@ -219,7 +219,7 @@ If you use another migration tool, apply both SQL migrations for your database i | PostgreSQL | [001 SQL](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/001__create_okapi_outbox_table.sql) | [002 SQL](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql) | | MySQL | [001 SQL](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/001__create_okapi_outbox_table.sql) | [002 SQL](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql) | -For an existing installation, apply `002` before starting updated workers. MySQL claims explicitly select the new index and fail if it is missing. PostgreSQL's `002` uses `CREATE INDEX CONCURRENTLY`, so run it outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. +For an existing installation, apply `002` before starting updated workers. MySQL claims explicitly select the new index and fail if it is missing. PostgreSQL's `002` uses `CREATE INDEX CONCURRENTLY`, so run it outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. `CREATE INDEX CONCURRENTLY` waits for every transaction in the database that started before it, not only those on `okapi_outbox`, so long-running transactions (including old workers delivering a claimed batch) delay it, and with Liquibase they delay application startup while the changelog lock is held. Run it when no long transactions are open, or apply it ahead of the rollout. okapi stores messages in the fixed `okapi_outbox` table. When the built-in Liquibase integration is used, it also uses two dedicated migration tracking tables: diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt index 2f92540..c4f76f6 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxProcessor.kt @@ -36,7 +36,10 @@ class OutboxProcessor @JvmOverloads constructor( fun processNext(limit: Int = 10): Int { val batchStart = clock.instant() val routeAwareStore = store as? RouteAwareOutboxStore - ?: error("OutboxProcessor requires a RouteAwareOutboxStore to avoid claiming unsupported delivery types") + ?: error( + "OutboxProcessor requires a RouteAwareOutboxStore to avoid claiming unsupported delivery types, " + + "got ${store.javaClass.name}. A wrapper or proxy around a store must implement RouteAwareOutboxStore itself.", + ) check(deliveryTypes.isNotEmpty()) { "OutboxProcessor requires at least one delivery type" } val claimed = if (limit <= 0) { emptyList() @@ -51,6 +54,8 @@ class OutboxProcessor @JvmOverloads constructor( first } else { val remaining = limit - first.size + // Stores order this claim by delivery type, so leftover capacity favours types that sort first. + // Each type is still claimed first once every N ticks, so none is starved. val other = routeAwareStore.claimPending(remainingTypes, remaining) check(other.size <= remaining && other.all { it.deliveryType in remainingTypes }) { "RouteAwareOutboxStore returned entries outside the requested types or limit" diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlOutboxStoreTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlOutboxStoreTest.kt index 0ecfce0..b4154b6 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlOutboxStoreTest.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlOutboxStoreTest.kt @@ -3,6 +3,7 @@ package com.softwaremill.okapi.test.store import com.softwaremill.okapi.mysql.MysqlOutboxStore import com.softwaremill.okapi.test.support.MysqlTestSupport import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.collections.shouldBeEmpty class MysqlOutboxStoreTest : FunSpec({ val db = MysqlTestSupport() @@ -19,4 +20,20 @@ class MysqlOutboxStoreTest : FunSpec({ stopDb = { db.stop() }, truncate = { db.truncate() }, ) + + test("[mysql] route-aware claim raises no server warnings") { + val store = MysqlOutboxStore(connectionProvider = db.jdbc) + val warnings = db.jdbc.withTransaction { + store.claimPending(setOf("kafka", "http"), 10) + db.jdbc.withConnection { conn -> + conn.createStatement().use { stmt -> + stmt.executeQuery("SHOW WARNINGS").use { rs -> + generateSequence { if (rs.next()) "${rs.getInt("Code")}: ${rs.getString("Message")}" else null }.toList() + } + } + } + } + + warnings.shouldBeEmpty() + } }) diff --git a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt index b9a90c3..0b097ec 100644 --- a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt +++ b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt @@ -72,14 +72,15 @@ class MysqlOutboxStore( if (deliveryTypes.isEmpty() || limit <= 0) return emptyList() val routes = deliveryTypes.sorted() val placeholders = routes.joinToString(",") { "?" } - val binaryPlaceholders = routes.joinToString(",") { "BINARY ?" } + val binaryPlaceholders = routes.joinToString(",") { "CAST(? AS BINARY)" } // A matching index avoids scanning and locking rows for other delivery types. // The binary predicate excludes case-insensitive collation matches for another route. + // CAST rather than the BINARY operator, which is deprecated since 8.0.27 and warns on every call. val sql = """ SELECT * FROM okapi_outbox FORCE INDEX (idx_okapi_outbox_status_delivery_created_id) WHERE status = ? AND delivery_type IN ($placeholders) - AND BINARY delivery_type IN ($binaryPlaceholders) + AND CAST(delivery_type AS BINARY) IN ($binaryPlaceholders) ORDER BY delivery_type ASC, created_at ASC, id ASC LIMIT ? FOR UPDATE SKIP LOCKED diff --git a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql index 1dc8a46..e42033f 100644 --- a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql +++ b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql @@ -1,6 +1,10 @@ --liquibase formatted sql --changeset outbox:002 runInTransaction:false +-- A failed or cancelled CREATE INDEX CONCURRENTLY leaves an INVALID index behind. Drop it first so a +-- retried changeset rebuilds the index instead of IF NOT EXISTS silently keeping the invalid one. +DROP INDEX CONCURRENTLY IF EXISTS idx_okapi_outbox_pending_delivery_created_id; + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_okapi_outbox_pending_delivery_created_id ON okapi_outbox (delivery_type, created_at, id) WHERE status = 'PENDING'; diff --git a/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt b/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt index 4a5ade8..1254bfd 100644 --- a/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt +++ b/okapi-spring-boot/src/main/kotlin/com/softwaremill/okapi/springboot/OutboxAutoConfiguration.kt @@ -195,8 +195,10 @@ class OutboxAutoConfiguration( "Define a MessageDeliverer or set okapi.processor.enabled=false for publisher-only use." } check(!processorEnabled || outboxStore is RouteAwareOutboxStore) { - "Okapi processor requires a RouteAwareOutboxStore. " + - "Implement route-aware claiming in the custom OutboxStore or set okapi.processor.enabled=false." + "Okapi processor requires a RouteAwareOutboxStore, but the OutboxStore bean is ${outboxStore.javaClass.name}. " + + "If it wraps or proxies a built-in store, implement RouteAwareOutboxStore on the wrapper and delegate " + + "claimPending(deliveryTypes, limit); otherwise implement route-aware claiming in the custom store, " + + "or set okapi.processor.enabled=false." } return OutboxProcessor( store = outboxStore,