Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,15 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with *

### Fixed

- **`ExposedTransactionRunner` no longer re-delivers a batch when its transaction fails.** Exposed
retries a transaction whose block throws an `SQLException` up to `DatabaseConfig.defaultMaxAttempts`
times (3 by default). Delivery is not transactional, so a failed status update or commit, such as a
deadlock victim or a connection reset, re-claimed and delivered the same entries again, up to three
times per tick. The runner now makes a single attempt, and the scheduler retries on its next tick.
The purger and the metrics reads, which use the same runner, also no longer retry within a run;
they try again on their next one. A runner call inside an existing transaction, and every other
Exposed transaction in the application, keep their configured retries.
([#134](https://github.com/softwaremill/okapi/issues/134))
- **MySQL: the processor no longer blocks `publish()` for the duration of delivery.** Under MySQL's
default `REPEATABLE READ`, the claim's `FOR UPDATE SKIP LOCKED` took gap locks on the claim index;
with a backlog smaller than the batch, the gap ran to the end of the claimed type's range, exactly
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package com.softwaremill.okapi.exposed
import com.softwaremill.okapi.core.TransactionRunner
import org.jetbrains.exposed.v1.core.vendors.MysqlDialect
import org.jetbrains.exposed.v1.jdbc.Database
import org.jetbrains.exposed.v1.jdbc.transactions.currentOrNull
import org.jetbrains.exposed.v1.jdbc.transactions.transaction
import org.jetbrains.exposed.v1.jdbc.transactions.transactionManager
import java.sql.Connection

/**
Expand All @@ -20,6 +22,14 @@ import java.sql.Connection
* `DatabaseConfig.defaultIsolationLevel` set for the rest of the application. Other databases keep
* their default (Postgres already defaults to READ COMMITTED).
*
* Each block runs once (issue #134). Exposed retries a top-level transaction whose block throws an
* `SQLException` up to `DatabaseConfig.defaultMaxAttempts` times (3 by default). For the processor
* that would re-claim and re-deliver the same entries on a failed status update or commit, since
* delivery is not transactional. The scheduler instead retries on its next tick. The same applies to
* the purger and metrics reads, which also retry on their next run. Inside an existing transaction
* the block joins it and the caller's retry settings are left alone, as are those of every other
* transaction in the application.
*
* @param database The [Database] instance where the outbox table resides.
*/
class ExposedTransactionRunner private constructor(
Expand All @@ -44,5 +54,17 @@ class ExposedTransactionRunner private constructor(
}
}

override fun <T> runInTransaction(block: () -> T): T = transaction(database, transactionIsolation = transactionIsolation) { block() }
override fun <T> runInTransaction(block: () -> T): T {
// Nested, Exposed reuses the caller's transaction by default, so only a top-level one is changed.
val topLevel = database.transactionManager.currentOrNull() == null
return transaction(database, transactionIsolation = transactionIsolation) {
if (topLevel) {
// Exposed reads these after a failed attempt, so setting them here applies to this one.
maxAttempts = 1
minRetryDelay = 0
maxRetryDelay = 0
}
block()
}
}
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,46 @@
package com.softwaremill.okapi.exposed

import com.softwaremill.okapi.core.DeliveryInfo
import com.softwaremill.okapi.core.DeliveryResult
import com.softwaremill.okapi.core.MessageDeliverer
import com.softwaremill.okapi.core.OutboxEntry
import com.softwaremill.okapi.core.OutboxEntryProcessor
import com.softwaremill.okapi.core.OutboxMessage
import com.softwaremill.okapi.core.OutboxProcessor
import com.softwaremill.okapi.core.OutboxStatus
import com.softwaremill.okapi.core.RetryPolicy
import com.softwaremill.okapi.core.RouteAwareOutboxStore
import io.kotest.assertions.throwables.shouldThrow
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.shouldBe
import org.jetbrains.exposed.v1.core.DatabaseConfig
import org.jetbrains.exposed.v1.jdbc.Database
import org.jetbrains.exposed.v1.jdbc.transactions.TransactionManager
import org.jetbrains.exposed.v1.jdbc.transactions.transaction
import org.jetbrains.exposed.v1.jdbc.transactions.transactionManager
import java.sql.Connection
import java.sql.SQLException
import java.time.Clock
import java.time.Instant
import java.util.concurrent.atomic.AtomicInteger

private val stubInfo = object : DeliveryInfo {
override val type = "stub"
override fun serialize() = "{}"
}

/** Hands out one PENDING entry per claim and fails every status update, like a deadlock victim. */
private class FailingUpdateStore : RouteAwareOutboxStore {
override fun persist(entry: OutboxEntry) = entry
override fun claimPending(limit: Int): List<OutboxEntry> = error("not used")
override fun claimPending(deliveryTypes: Set<String>, limit: Int, now: Instant): List<OutboxEntry> =
listOf(OutboxEntry.createPending(OutboxMessage("e", "{}"), stubInfo, Instant.EPOCH))
override fun updateAfterProcessing(entry: OutboxEntry): OutboxEntry = throw SQLException("Deadlock found", "40001")
override fun updateAfterProcessingBatch(entries: List<OutboxEntry>): List<OutboxEntry> = throw SQLException("Deadlock found", "40001")
override fun removeDeliveredBefore(time: Instant, limit: Int) = 0
override fun findOldestCreatedAt(statuses: Set<OutboxStatus>) = emptyMap<OutboxStatus, Instant>()
override fun countByStatuses() = emptyMap<OutboxStatus, Long>()
}

class ExposedTransactionRunnerTest : FunSpec({

Expand Down Expand Up @@ -44,4 +78,63 @@ class ExposedTransactionRunnerTest : FunSpec({
test("null falls back to the database's default isolation") {
isolationSeenBy(ExposedTransactionRunner(db, null)) shouldBe db.transactionManager.defaultIsolationLevel
}

// Issue #134: Exposed's default retry would re-run claim + delivery on an SQLException.
test("runs a block that throws SQLException once, whatever the database's default max attempts") {
val retryingDb = Database.connect(
"jdbc:h2:mem:runner_test_retry_${System.nanoTime()};DB_CLOSE_DELAY=-1",
driver = "org.h2.Driver",
databaseConfig = DatabaseConfig { defaultMaxAttempts = 5 },
)
val runs = AtomicInteger()

shouldThrow<SQLException> {
ExposedTransactionRunner(retryingDb).runInTransaction {
runs.incrementAndGet()
throw SQLException("connection reset", "08S01")
}
}

runs.get() shouldBe 1
}

test("a failed status update does not deliver the batch again") {
val deliveries = AtomicInteger()
val processor = OutboxProcessor(
store = FailingUpdateStore(),
entryProcessor = OutboxEntryProcessor(
object : MessageDeliverer {
override val type = stubInfo.type
override fun deliver(entry: OutboxEntry): DeliveryResult {
deliveries.incrementAndGet()
return DeliveryResult.Success
}
},
RetryPolicy(maxRetries = 1),
Clock.systemUTC(),
),
)

shouldThrow<SQLException> { ExposedTransactionRunner(db).runInTransaction { processor.processBatch(10) } }

deliveries.get() shouldBe 1
}

test("a nested call leaves the caller's transaction retries alone") {
val attempts = AtomicInteger()

shouldThrow<SQLException> {
transaction(db) {
attempts.incrementAndGet()
ExposedTransactionRunner(db).runInTransaction { }
throw SQLException("connection reset", "08S01")
}
}

attempts.get() shouldBe db.transactionManager.defaultMaxAttempts
}

test("a block that succeeds commits and returns its result") {
ExposedTransactionRunner(db).runInTransaction { 42 } shouldBe 42
}
})