diff --git a/CHANGELOG.md b/CHANGELOG.md index f68768c..9f00b06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,15 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * ### Fixed +- **`ExposedTransactionRunner` no longer re-delivers a batch when its transaction fails.** Exposed + retries a transaction whose block throws an `SQLException` up to `DatabaseConfig.defaultMaxAttempts` + times (3 by default). Delivery is not transactional, so a failed status update or commit, such as a + deadlock victim or a connection reset, re-claimed and delivered the same entries again, up to three + times per tick. The runner now makes a single attempt, and the scheduler retries on its next tick. + The purger and the metrics reads, which use the same runner, also no longer retry within a run; + they try again on their next one. A runner call inside an existing transaction, and every other + Exposed transaction in the application, keep their configured retries. + ([#134](https://github.com/softwaremill/okapi/issues/134)) - **MySQL: the processor no longer blocks `publish()` for the duration of delivery.** Under MySQL's default `REPEATABLE READ`, the claim's `FOR UPDATE SKIP LOCKED` took gap locks on the claim index; with a backlog smaller than the batch, the gap ran to the end of the claimed type's range, exactly diff --git a/okapi-exposed/src/main/kotlin/com/softwaremill/okapi/exposed/ExposedTransactionRunner.kt b/okapi-exposed/src/main/kotlin/com/softwaremill/okapi/exposed/ExposedTransactionRunner.kt index 5740daf..74c0428 100644 --- a/okapi-exposed/src/main/kotlin/com/softwaremill/okapi/exposed/ExposedTransactionRunner.kt +++ b/okapi-exposed/src/main/kotlin/com/softwaremill/okapi/exposed/ExposedTransactionRunner.kt @@ -3,7 +3,9 @@ package com.softwaremill.okapi.exposed import com.softwaremill.okapi.core.TransactionRunner import org.jetbrains.exposed.v1.core.vendors.MysqlDialect import org.jetbrains.exposed.v1.jdbc.Database +import org.jetbrains.exposed.v1.jdbc.transactions.currentOrNull import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.transactions.transactionManager import java.sql.Connection /** @@ -20,6 +22,14 @@ import java.sql.Connection * `DatabaseConfig.defaultIsolationLevel` set for the rest of the application. Other databases keep * their default (Postgres already defaults to READ COMMITTED). * + * Each block runs once (issue #134). Exposed retries a top-level transaction whose block throws an + * `SQLException` up to `DatabaseConfig.defaultMaxAttempts` times (3 by default). For the processor + * that would re-claim and re-deliver the same entries on a failed status update or commit, since + * delivery is not transactional. The scheduler instead retries on its next tick. The same applies to + * the purger and metrics reads, which also retry on their next run. Inside an existing transaction + * the block joins it and the caller's retry settings are left alone, as are those of every other + * transaction in the application. + * * @param database The [Database] instance where the outbox table resides. */ class ExposedTransactionRunner private constructor( @@ -44,5 +54,17 @@ class ExposedTransactionRunner private constructor( } } - override fun runInTransaction(block: () -> T): T = transaction(database, transactionIsolation = transactionIsolation) { block() } + override fun runInTransaction(block: () -> T): T { + // Nested, Exposed reuses the caller's transaction by default, so only a top-level one is changed. + val topLevel = database.transactionManager.currentOrNull() == null + return transaction(database, transactionIsolation = transactionIsolation) { + if (topLevel) { + // Exposed reads these after a failed attempt, so setting them here applies to this one. + maxAttempts = 1 + minRetryDelay = 0 + maxRetryDelay = 0 + } + block() + } + } } diff --git a/okapi-exposed/src/test/kotlin/com/softwaremill/okapi/exposed/ExposedTransactionRunnerTest.kt b/okapi-exposed/src/test/kotlin/com/softwaremill/okapi/exposed/ExposedTransactionRunnerTest.kt index adb07c0..b812c3e 100644 --- a/okapi-exposed/src/test/kotlin/com/softwaremill/okapi/exposed/ExposedTransactionRunnerTest.kt +++ b/okapi-exposed/src/test/kotlin/com/softwaremill/okapi/exposed/ExposedTransactionRunnerTest.kt @@ -1,12 +1,46 @@ package com.softwaremill.okapi.exposed +import com.softwaremill.okapi.core.DeliveryInfo +import com.softwaremill.okapi.core.DeliveryResult +import com.softwaremill.okapi.core.MessageDeliverer +import com.softwaremill.okapi.core.OutboxEntry +import com.softwaremill.okapi.core.OutboxEntryProcessor +import com.softwaremill.okapi.core.OutboxMessage +import com.softwaremill.okapi.core.OutboxProcessor +import com.softwaremill.okapi.core.OutboxStatus +import com.softwaremill.okapi.core.RetryPolicy +import com.softwaremill.okapi.core.RouteAwareOutboxStore +import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe import org.jetbrains.exposed.v1.core.DatabaseConfig import org.jetbrains.exposed.v1.jdbc.Database import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager +import org.jetbrains.exposed.v1.jdbc.transactions.transaction import org.jetbrains.exposed.v1.jdbc.transactions.transactionManager import java.sql.Connection +import java.sql.SQLException +import java.time.Clock +import java.time.Instant +import java.util.concurrent.atomic.AtomicInteger + +private val stubInfo = object : DeliveryInfo { + override val type = "stub" + override fun serialize() = "{}" +} + +/** Hands out one PENDING entry per claim and fails every status update, like a deadlock victim. */ +private class FailingUpdateStore : RouteAwareOutboxStore { + override fun persist(entry: OutboxEntry) = entry + override fun claimPending(limit: Int): List = error("not used") + override fun claimPending(deliveryTypes: Set, limit: Int, now: Instant): List = + listOf(OutboxEntry.createPending(OutboxMessage("e", "{}"), stubInfo, Instant.EPOCH)) + override fun updateAfterProcessing(entry: OutboxEntry): OutboxEntry = throw SQLException("Deadlock found", "40001") + override fun updateAfterProcessingBatch(entries: List): List = throw SQLException("Deadlock found", "40001") + override fun removeDeliveredBefore(time: Instant, limit: Int) = 0 + override fun findOldestCreatedAt(statuses: Set) = emptyMap() + override fun countByStatuses() = emptyMap() +} class ExposedTransactionRunnerTest : FunSpec({ @@ -44,4 +78,63 @@ class ExposedTransactionRunnerTest : FunSpec({ test("null falls back to the database's default isolation") { isolationSeenBy(ExposedTransactionRunner(db, null)) shouldBe db.transactionManager.defaultIsolationLevel } + + // Issue #134: Exposed's default retry would re-run claim + delivery on an SQLException. + test("runs a block that throws SQLException once, whatever the database's default max attempts") { + val retryingDb = Database.connect( + "jdbc:h2:mem:runner_test_retry_${System.nanoTime()};DB_CLOSE_DELAY=-1", + driver = "org.h2.Driver", + databaseConfig = DatabaseConfig { defaultMaxAttempts = 5 }, + ) + val runs = AtomicInteger() + + shouldThrow { + ExposedTransactionRunner(retryingDb).runInTransaction { + runs.incrementAndGet() + throw SQLException("connection reset", "08S01") + } + } + + runs.get() shouldBe 1 + } + + test("a failed status update does not deliver the batch again") { + val deliveries = AtomicInteger() + val processor = OutboxProcessor( + store = FailingUpdateStore(), + entryProcessor = OutboxEntryProcessor( + object : MessageDeliverer { + override val type = stubInfo.type + override fun deliver(entry: OutboxEntry): DeliveryResult { + deliveries.incrementAndGet() + return DeliveryResult.Success + } + }, + RetryPolicy(maxRetries = 1), + Clock.systemUTC(), + ), + ) + + shouldThrow { ExposedTransactionRunner(db).runInTransaction { processor.processBatch(10) } } + + deliveries.get() shouldBe 1 + } + + test("a nested call leaves the caller's transaction retries alone") { + val attempts = AtomicInteger() + + shouldThrow { + transaction(db) { + attempts.incrementAndGet() + ExposedTransactionRunner(db).runInTransaction { } + throw SQLException("connection reset", "08S01") + } + } + + attempts.get() shouldBe db.transactionManager.defaultMaxAttempts + } + + test("a block that succeeds commits and returns its result") { + ExposedTransactionRunner(db).runInTransaction { 42 } shouldBe 42 + } })