diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e6f5b1..94178b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,25 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * automatically. Cross-type claim order is no longer globally oldest first: the processor rotates the starting type, and any leftover batch capacity goes to the other types alphabetically. ([#116](https://github.com/softwaremill/okapi/issues/116)) +- **One unpersistable `lastError` no longer poisons its whole batch.** All results of a batch are + written in one update inside the claim transaction, so a `last_error` value the database rejected + rolled back the entire batch: entries that had already been delivered went back to `PENDING` with + `retries` unchanged and were redelivered on every tick, indefinitely (three times per tick under + Exposed's default retry). Two such values were easy to hit: an HTTP error body over 64 KB + (MySQL `TEXT` overflow, `ERROR 1406`) and a NUL character (refused by Postgres). + `OutboxEntry.retry`/`toFailed` now strip NULs and truncate to `OutboxEntry.MAX_LAST_ERROR_LENGTH` + (4096 chars, never splitting a surrogate pair), so every transport is covered. Both built-in + stores apply the same `OutboxEntry.sanitizeLastError` again when writing, so an entry built with + `copy(lastError = ...)` is covered too; custom stores should call it before writing `last_error`. + (#130) +- **`HttpMessageDeliverer` no longer buffers whole response bodies, or waits on them forever.** + Success bodies are discarded, and only the first `HttpMessageDeliverer.MAX_ERROR_BODY_BYTES` + (2 KB) of an error body is kept for `lastError`. At most `MAX_DRAINED_BODY_BYTES` (64 KB) of any + body is read: up to that the rest is drained so the connection is reused, past it the body is + cancelled. Each exchange, body included, must also finish within `EXCHANGE_TIMEOUT` (30 s) or it + is cancelled as a `RetriableFailure`; the request timeout only covers waiting for the headers. + Previously every response was read fully into heap, and a server dripping its body held delivery, + and with it the scheduler's claim transaction, open indefinitely. (#130) ### Changed diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntry.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntry.kt index 53b0aa3..176e455 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntry.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntry.kt @@ -28,7 +28,7 @@ data class OutboxEntry( updatedAt = now, retries = retries + 1, lastAttempt = now, - lastError = lastError, + lastError = sanitizeLastError(lastError), ) /** Returns a copy marked as permanently failed. */ @@ -36,7 +36,7 @@ data class OutboxEntry( status = OutboxStatus.FAILED, updatedAt = now, lastAttempt = now, - lastError = lastError, + lastError = sanitizeLastError(lastError), ) /** Returns a copy marked as successfully delivered. */ @@ -47,6 +47,39 @@ data class OutboxEntry( ) companion object { + /** + * Upper bound, in UTF-16 chars, on a stored [lastError]. A UTF-16 char encodes to at most + * 3 bytes of UTF-8 (a 4-byte code point takes two chars), so the value is at most ~12 KB, + * well inside MySQL's 65,535-byte `TEXT` column. + */ + const val MAX_LAST_ERROR_LENGTH: Int = 4096 + + /** Appended to a [lastError] (or an HTTP error body) that was cut short. */ + const val TRUNCATION_MARKER: String = "... [truncated]" + + /** + * Makes [error] safe to persist (issue #130). Every result of a batch is written in one + * update inside the claim transaction, so a single value the database rejects rolls back the + * whole batch -- including entries that were already delivered -- without advancing + * `retries`, and the next tick redelivers all of them, forever. Two such values are cheap to + * produce: an over-long one (an HTTP error page in the response body overflows MySQL's + * `TEXT`) and one containing NUL, which Postgres refuses in text. NULs are dropped and the + * rest is cut to [MAX_LAST_ERROR_LENGTH], never between the two halves of a surrogate pair, + * which MySQL would reject as an invalid string. + * + * [retry] and [toFailed] apply it, and the built-in stores apply it again when writing, so an + * entry built with `copy(lastError = ...)` is covered too. Custom [OutboxStore]s should call + * it before writing `last_error`. It is idempotent. + */ + @JvmStatic + fun sanitizeLastError(error: String): String { + val withoutNul = if (error.indexOf('\u0000') >= 0) error.replace("\u0000", "") else error + if (withoutNul.length <= MAX_LAST_ERROR_LENGTH) return withoutNul + var end = MAX_LAST_ERROR_LENGTH - TRUNCATION_MARKER.length + if (Character.isHighSurrogate(withoutNul[end - 1])) end-- + return withoutNul.substring(0, end) + TRUNCATION_MARKER + } + /** Creates a new PENDING entry from a [message] and [deliveryInfo]. */ @JvmStatic fun createPending(message: OutboxMessage, deliveryInfo: DeliveryInfo, now: Instant): OutboxEntry = diff --git a/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxEntryLastErrorTest.kt b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxEntryLastErrorTest.kt new file mode 100644 index 0000000..46f0e28 --- /dev/null +++ b/okapi-core/src/test/kotlin/com/softwaremill/okapi/core/OutboxEntryLastErrorTest.kt @@ -0,0 +1,53 @@ +package com.softwaremill.okapi.core + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.ints.shouldBeLessThanOrEqual +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldEndWith +import io.kotest.matchers.string.shouldNotContain +import io.kotest.matchers.string.shouldStartWith +import java.time.Instant + +class OutboxEntryLastErrorTest : FunSpec({ + val info = object : DeliveryInfo { + override val type = "stub" + override fun serialize(): String = """{"type":"stub"}""" + } + val entry = OutboxEntry.createPending(OutboxMessage("test.event", "{}"), info, Instant.EPOCH) + val now = Instant.parse("2024-01-01T00:00:00Z") + + test("a short error is stored unchanged") { + entry.retry(now, "HTTP 503: unavailable").lastError shouldBe "HTTP 503: unavailable" + entry.toFailed(now, "HTTP 400: bad").lastError shouldBe "HTTP 400: bad" + } + + test("an over-long error is truncated to MAX_LAST_ERROR_LENGTH with a marker, on retry and on failure") { + val huge = "HTTP 503: " + "x".repeat(100_000) + listOf(entry.retry(now, huge), entry.toFailed(now, huge)).forEach { updated -> + val stored = checkNotNull(updated.lastError) + stored.length shouldBeLessThanOrEqual OutboxEntry.MAX_LAST_ERROR_LENGTH + stored shouldStartWith "HTTP 503: xxx" + stored shouldEndWith OutboxEntry.TRUNCATION_MARKER + } + } + + test("an error of exactly MAX_LAST_ERROR_LENGTH is not truncated") { + val exact = "y".repeat(OutboxEntry.MAX_LAST_ERROR_LENGTH) + entry.retry(now, exact).lastError shouldBe exact + } + + test("NUL characters are removed") { + entry.retry(now, "binary\u0000body\u0000").lastError shouldBe "binarybody" + } + + test("truncation never splits a surrogate pair") { + // Pad so the cut lands right after the high surrogate of an emoji. + val cut = OutboxEntry.MAX_LAST_ERROR_LENGTH - OutboxEntry.TRUNCATION_MARKER.length + val error = "a".repeat(cut - 1) + "๐Ÿ˜€" + "b".repeat(10_000) + val stored = checkNotNull(entry.retry(now, error).lastError) + stored.toCharArray().forEachIndexed { i, c -> + if (Character.isHighSurrogate(c)) Character.isLowSurrogate(stored.getOrElse(i + 1) { ' ' }) shouldBe true + } + stored shouldNotContain "\uD83D..." + } +}) diff --git a/okapi-http/src/main/kotlin/com/softwaremill/okapi/http/HttpMessageDeliverer.kt b/okapi-http/src/main/kotlin/com/softwaremill/okapi/http/HttpMessageDeliverer.kt index 96ace8f..2034682 100644 --- a/okapi-http/src/main/kotlin/com/softwaremill/okapi/http/HttpMessageDeliverer.kt +++ b/okapi-http/src/main/kotlin/com/softwaremill/okapi/http/HttpMessageDeliverer.kt @@ -6,13 +6,20 @@ import com.softwaremill.okapi.core.DeliveryResult import com.softwaremill.okapi.core.MessageDeliverer import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxHeaders +import java.io.ByteArrayOutputStream import java.io.IOException import java.net.URI import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.nio.ByteBuffer import java.time.Duration import java.util.concurrent.CompletableFuture +import java.util.concurrent.CompletionException +import java.util.concurrent.CompletionStage +import java.util.concurrent.Flow +import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeoutException import javax.net.ssl.SSLException /** @@ -37,23 +44,29 @@ import javax.net.ssl.SSLException * sequentially on [HttpClient.send]. The JDK [HttpClient] will use HTTP/2 multiplexing when * supported by the server; otherwise it will fall back to HTTP/1.1 (potentially with multiple * connections) โ€” either way, requests can be overlapped without explicit per-host grouping. + * + * Each exchange, response body included, must complete within [EXCHANGE_TIMEOUT]; otherwise it is + * cancelled and classified as [DeliveryResult.RetriableFailure]. [HttpRequest.timeout] alone only + * covers waiting for the response headers, so a server dripping a body could otherwise hold + * delivery, and with it the scheduler's claim transaction, open indefinitely. */ -class HttpMessageDeliverer @JvmOverloads constructor( +class HttpMessageDeliverer internal constructor( private val urlResolver: ServiceUrlResolver, - private val httpClient: HttpClient = defaultHttpClient(), - private val retriableStatusCodes: Set = DEFAULT_RETRIABLE_STATUS_CODES, + private val httpClient: HttpClient, + private val retriableStatusCodes: Set, + private val exchangeTimeout: Duration, ) : MessageDeliverer { + @JvmOverloads constructor( + urlResolver: ServiceUrlResolver, + httpClient: HttpClient = defaultHttpClient(), + retriableStatusCodes: Set = DEFAULT_RETRIABLE_STATUS_CODES, + ) : this(urlResolver, httpClient, retriableStatusCodes, EXCHANGE_TIMEOUT) + override val type: String = HttpDeliveryInfo.TYPE - override fun deliver(entry: OutboxEntry): DeliveryResult = try { - val request = buildRequest(entry) - val response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) - classifyResponse(response.statusCode(), response.body()) - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - classifyThrowable(e) - } catch (e: Exception) { - classifyThrowable(e) + override fun deliver(entry: OutboxEntry): DeliveryResult = when (val attempt = fireOne(entry)) { + is SendAttempt.ImmediateFailure -> attempt.result + is SendAttempt.InFlight -> awaitResult(attempt) } /** @@ -70,7 +83,7 @@ class HttpMessageDeliverer @JvmOverloads constructor( return attempts.map { (entry, attempt) -> val result = when (attempt) { is SendAttempt.ImmediateFailure -> attempt.result - is SendAttempt.InFlight -> awaitResult(attempt.future) + is SendAttempt.InFlight -> awaitResult(attempt) } DeliveryOutcome(entry, result) } @@ -80,12 +93,14 @@ class HttpMessageDeliverer @JvmOverloads constructor( * Awaits via [CompletableFuture.get] rather than [CompletableFuture.join] so that an * interrupt on the caller (processor thread shutdown/backpressure) is observed instead of * being ignored โ€” the flag is restored here, on the interrupted thread itself, and the - * remaining not-yet-awaited futures in the batch will then also fail fast as retriable. + * remaining not-yet-awaited futures in the batch will then also fail fast as retriable. The + * interrupted exchange itself is cancelled, as [HttpClient.send] would. */ - private fun awaitResult(future: CompletableFuture): DeliveryResult = try { - future.get() + private fun awaitResult(attempt: SendAttempt.InFlight): DeliveryResult = try { + attempt.result.get() } catch (e: InterruptedException) { Thread.currentThread().interrupt() + attempt.exchange.cancel(true) classifyThrowable(e) } catch (e: Exception) { // fireOne()'s .exceptionally already converts every failure into a normal completion, so @@ -94,13 +109,25 @@ class HttpMessageDeliverer @JvmOverloads constructor( classifyThrowable(e.cause ?: e) } + /** + * The deadline runs on a copy of the exchange future: completing that copy with a + * [TimeoutException] does not abort the HTTP exchange, so the exchange is cancelled explicitly. + */ private fun fireOne(entry: OutboxEntry): SendAttempt = try { val request = buildRequest(entry) - val future: CompletableFuture = httpClient - .sendAsync(request, HttpResponse.BodyHandlers.ofString()) - .thenApply { response -> classifyResponse(response.statusCode(), response.body()) } - .exceptionally { e -> classifyThrowable(e.cause ?: e) } - SendAttempt.InFlight(future) + val exchange = httpClient.sendAsync(request, boundedBodyHandler) + val result: CompletableFuture = exchange.copy() + .orTimeout(exchangeTimeout.toMillis(), TimeUnit.MILLISECONDS) + .handle { response, failure -> + if (failure == null) { + classifyResponse(response.statusCode(), response.body()) + } else { + val cause = if (failure is CompletionException) failure.cause ?: failure else failure + if (cause is TimeoutException) exchange.cancel(true) + classifyThrowable(cause) + } + } + SendAttempt.InFlight(exchange, result) } catch (e: Exception) { SendAttempt.ImmediateFailure(classifyThrowable(e)) } @@ -129,6 +156,18 @@ class HttpMessageDeliverer @JvmOverloads constructor( .build() } + /** + * Discards success bodies and keeps at most [MAX_ERROR_BODY_BYTES] of an error body (issue #130). + * The body only ever feeds `lastError`, and `BodyHandlers.ofString()` buffered all of it in heap + * -- an endpoint answering with a multi-megabyte error page cost that much memory per in-flight + * request, and the full text then overflowed the `last_error` column. Either way at most + * [MAX_DRAINED_BODY_BYTES] are read; see [BoundedUtf8BodySubscriber]. + */ + private val boundedBodyHandler = HttpResponse.BodyHandler { info -> + val keep = if (info.statusCode() in 200..299) 0 else MAX_ERROR_BODY_BYTES + BoundedUtf8BodySubscriber(keep, MAX_DRAINED_BODY_BYTES) + } + private fun classifyResponse(status: Int, body: String): DeliveryResult = when { status in 200..299 -> DeliveryResult.Success status in retriableStatusCodes -> DeliveryResult.RetriableFailure("HTTP $status: $body") @@ -142,6 +181,7 @@ class HttpMessageDeliverer @JvmOverloads constructor( is JsonProcessingException -> DeliveryResult.PermanentFailure(message) is SSLException -> DeliveryResult.PermanentFailure(message) is IOException -> DeliveryResult.RetriableFailure(message) + is TimeoutException -> DeliveryResult.RetriableFailure("HTTP exchange did not complete within $exchangeTimeout") // Deliberately side-effect free: this can run on an HttpClient completion thread // (via fireOne()'s .exceptionally), not the caller's thread, so interrupting // "current thread" here would interrupt an unrelated shared pool thread. Callers on @@ -152,11 +192,25 @@ class HttpMessageDeliverer @JvmOverloads constructor( } private sealed interface SendAttempt { - data class InFlight(val future: CompletableFuture) : SendAttempt + /** [exchange] is the raw [HttpClient.sendAsync] future, kept so the exchange can be cancelled. */ + data class InFlight(val exchange: CompletableFuture<*>, val result: CompletableFuture) : SendAttempt data class ImmediateFailure(val result: DeliveryResult) : SendAttempt } companion object { + /** How much of an error response body is kept for `lastError`; the rest is dropped. */ + const val MAX_ERROR_BODY_BYTES: Int = 2048 + + /** + * How much of any response body is read. Up to this size the body is drained so the + * connection can be reused; past it the body is cancelled, which closes an HTTP/1.1 + * connection (HTTP/2 resets only the stream). + */ + const val MAX_DRAINED_BODY_BYTES: Long = 64 * 1024 + + /** Upper bound on one exchange, from sending the request to the end of the response body. */ + val EXCHANGE_TIMEOUT: Duration = Duration.ofSeconds(30) + val DEFAULT_RETRIABLE_STATUS_CODES: Set = (500..599).toSet() + 429 + 408 private val CONNECT_TIMEOUT: Duration = Duration.ofSeconds(10) private val REQUEST_TIMEOUT: Duration = Duration.ofSeconds(30) @@ -167,3 +221,59 @@ class HttpMessageDeliverer @JvmOverloads constructor( .build() } } + +/** + * Keeps the first [keepLimit] bytes of a body as UTF-8 and reads at most [readLimit] bytes in total. + * Below [readLimit] the rest is drained without being stored, so the connection stays reusable; + * past it the subscription is cancelled rather than reading an arbitrarily large or endless body. + * A multi-byte character cut at [keepLimit] decodes as U+FFFD, which is fine for an error message. + * + * The HTTP client signals this subscriber serially (Reactive Streams ยง1.3), so its state needs no + * synchronization. + */ +private class BoundedUtf8BodySubscriber( + private val keepLimit: Int, + private val readLimit: Long, +) : HttpResponse.BodySubscriber { + private val body = CompletableFuture() + private val kept = ByteArrayOutputStream() + private var read = 0L + private var done = false + private lateinit var subscription: Flow.Subscription + + override fun getBody(): CompletionStage = body + + override fun onSubscribe(subscription: Flow.Subscription) { + this.subscription = subscription + subscription.request(Long.MAX_VALUE) + } + + override fun onNext(item: List) { + if (done) return + for (buffer in item) { + val take = minOf(keepLimit - kept.size(), buffer.remaining()) + if (take > 0) kept.write(ByteArray(take).also { buffer.get(it) }) + read += take + buffer.remaining() + } + if (read > readLimit) { + subscription.cancel() + finish() + } + } + + override fun onError(throwable: Throwable) { + if (done) return + done = true + body.completeExceptionally(throwable) + } + + override fun onComplete() { + if (!done) finish() + } + + private fun finish() { + done = true + val truncated = read > kept.size() + body.complete(kept.toString(Charsets.UTF_8) + if (truncated && keepLimit > 0) OutboxEntry.TRUNCATION_MARKER else "") + } +} diff --git a/okapi-http/src/test/kotlin/com/softwaremill/okapi/http/HttpMessageDelivererTest.kt b/okapi-http/src/test/kotlin/com/softwaremill/okapi/http/HttpMessageDelivererTest.kt index 9d79ccd..f199f57 100644 --- a/okapi-http/src/test/kotlin/com/softwaremill/okapi/http/HttpMessageDelivererTest.kt +++ b/okapi-http/src/test/kotlin/com/softwaremill/okapi/http/HttpMessageDelivererTest.kt @@ -14,8 +14,14 @@ import com.softwaremill.okapi.core.OutboxMessage import io.kotest.core.spec.style.FunSpec import io.kotest.datatest.WithDataTestName import io.kotest.datatest.withTests +import io.kotest.matchers.ints.shouldBeLessThanOrEqual +import io.kotest.matchers.longs.shouldBeLessThan import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldEndWith +import io.kotest.matchers.string.shouldStartWith import io.kotest.matchers.types.shouldBeInstanceOf +import java.net.http.HttpClient +import java.time.Duration import java.time.Instant import kotlin.reflect.KClass @@ -132,4 +138,77 @@ class HttpMessageDelivererTest : FunSpec({ val d = HttpMessageDeliverer(throwingResolver) d.deliver(entry()).shouldBeInstanceOf() } + + context("response bodies are bounded (#130)") { + val hugeErrorPage = "" + "x".repeat(1_000_000) + "" + val maxErrorLength = "HTTP 503: ".length + HttpMessageDeliverer.MAX_ERROR_BODY_BYTES + OutboxEntry.TRUNCATION_MARKER.length + + test("deliver() keeps only MAX_ERROR_BODY_BYTES of an error body") { + wiremock.stubFor(post(urlEqualTo("/test")).willReturn(aResponse().withStatus(503).withBody(hugeErrorPage))) + val error = deliverer.deliver(entry()).shouldBeInstanceOf().error + error shouldStartWith "HTTP 503: xxx" + error shouldEndWith OutboxEntry.TRUNCATION_MARKER + error.length shouldBeLessThanOrEqual maxErrorLength + } + + test("deliverBatch() keeps only MAX_ERROR_BODY_BYTES of an error body") { + wiremock.stubFor(post(urlEqualTo("/test")).willReturn(aResponse().withStatus(400).withBody(hugeErrorPage))) + val outcomes = deliverer.deliverBatch(listOf(entry(), entry())) + outcomes.forEach { outcome -> + val error = outcome.result.shouldBeInstanceOf().error + error shouldEndWith OutboxEntry.TRUNCATION_MARKER + error.length shouldBeLessThanOrEqual maxErrorLength + } + } + + test("a short error body is kept whole") { + wiremock.stubFor(post(urlEqualTo("/test")).willReturn(aResponse().withStatus(500).withBody("Internal Server Error"))) + deliverer.deliver(entry()) shouldBe DeliveryResult.RetriableFailure("HTTP 500: Internal Server Error") + } + + test("a large success body is discarded and still counts as Success") { + wiremock.stubFor(post(urlEqualTo("/test")).willReturn(aResponse().withStatus(200).withBody(hugeErrorPage))) + deliverer.deliver(entry()) shouldBe DeliveryResult.Success + deliverer.deliverBatch(listOf(entry())).single().result shouldBe DeliveryResult.Success + } + + fun elapsedMillis(block: () -> Unit): Long { + val start = System.nanoTime() + block() + return (System.nanoTime() - start) / 1_000_000 + } + + test("a slow body is not read past MAX_DRAINED_BODY_BYTES") { + // 1 MB in 100 chunks over 20 s: reading it all would take 20 s, the read cap is hit in ~1.5 s. + wiremock.stubFor( + post(urlEqualTo("/test")).willReturn( + aResponse().withStatus(200).withBody(hugeErrorPage).withChunkedDribbleDelay(100, 20_000), + ), + ) + elapsedMillis { deliverer.deliver(entry()) shouldBe DeliveryResult.Success } shouldBeLessThan 10_000 + elapsedMillis { + deliverer.deliverBatch(listOf(entry())).single().result shouldBe DeliveryResult.Success + } shouldBeLessThan 10_000 + } + + test("an exchange whose body outlasts the exchange timeout is cancelled as a RetriableFailure") { + // A small error body dripped over 20 s: under the read cap, so only the deadline stops it. + wiremock.stubFor( + post(urlEqualTo("/test")).willReturn( + aResponse().withStatus(503).withBody("x".repeat(1_000)).withChunkedDribbleDelay(20, 20_000), + ), + ) + val impatient = HttpMessageDeliverer( + ServiceUrlResolver { "http://localhost:${wiremock.port()}" }, + HttpClient.newHttpClient(), + HttpMessageDeliverer.DEFAULT_RETRIABLE_STATUS_CODES, + Duration.ofSeconds(1), + ) + val expected = DeliveryResult.RetriableFailure("HTTP exchange did not complete within PT1S") + elapsedMillis { impatient.deliver(entry()) shouldBe expected } shouldBeLessThan 10_000 + elapsedMillis { + impatient.deliverBatch(listOf(entry(), entry())).map { it.result } shouldBe listOf(expected, expected) + } shouldBeLessThan 10_000 + } + } }) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/LastErrorPoisonBatchTests.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/LastErrorPoisonBatchTests.kt new file mode 100644 index 0000000..7cea37c --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/LastErrorPoisonBatchTests.kt @@ -0,0 +1,123 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.core.DeliveryInfo +import com.softwaremill.okapi.core.DeliveryResult +import com.softwaremill.okapi.core.MessageDeliverer +import com.softwaremill.okapi.core.OutboxEntry +import com.softwaremill.okapi.core.OutboxEntryProcessor +import com.softwaremill.okapi.core.OutboxMessage +import com.softwaremill.okapi.core.OutboxProcessor +import com.softwaremill.okapi.core.OutboxStatus +import com.softwaremill.okapi.core.OutboxStore +import com.softwaremill.okapi.core.RetryPolicy +import com.softwaremill.okapi.test.support.JdbcConnectionProvider +import io.kotest.assertions.throwables.shouldThrowAny +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.ints.shouldBeLessThanOrEqual +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldNotContain +import java.time.Clock + +private object PoisonTestDeliveryInfo : DeliveryInfo { + override val type: String = "poison-test" + override fun serialize(): String = """{"type":"poison-test"}""" +} + +/** Delivers `healthy` entries and fails `poisoned` ones with [error]. */ +private class SplitDeliverer(private val error: String) : MessageDeliverer { + override val type: String = PoisonTestDeliveryInfo.type + override fun deliver(entry: OutboxEntry): DeliveryResult = + if (entry.messageType == "healthy") DeliveryResult.Success else DeliveryResult.RetriableFailure(error) +} + +private data class Row(val status: OutboxStatus, val retries: Int, val lastError: String?) + +/** + * Regression tests for issue #130. Every result of a batch is persisted in one update inside the + * claim transaction, so a `lastError` the database rejects used to roll back the whole batch: + * already-delivered entries went back to PENDING with `retries` unchanged and were redelivered on + * every tick, forever. + * + * [rejectedError] is a value this database refuses in `last_error` (over-long for MySQL's `TEXT`, + * NUL for Postgres). The control test writes it with plain JDBC, bypassing the store, to prove it + * really is rejected, so the other tests passing means the value is sanitized, not that the + * database happened to accept it. + */ +fun FunSpec.lastErrorPoisonBatchTests( + dbName: String, + jdbcProvider: () -> JdbcConnectionProvider, + storeFactory: () -> OutboxStore, + startDb: () -> Unit, + stopDb: () -> Unit, + truncate: () -> Unit, + rejectedError: String, +) { + val clock = Clock.systemUTC() + + beforeSpec { startDb() } + afterSpec { stopDb() } + beforeEach { truncate() } + + fun seed(messageType: String): OutboxEntry { + val entry = OutboxEntry.createPending(OutboxMessage(messageType, "{}"), PoisonTestDeliveryInfo, clock.instant()) + jdbcProvider().withTransaction { storeFactory().persist(entry) } + return entry + } + + fun row(messageType: String): Row = jdbcProvider().withTransaction { + jdbcProvider().withConnection { conn -> + conn.prepareStatement("SELECT status, retries, last_error FROM okapi_outbox WHERE message_type = ?").use { stmt -> + stmt.setString(1, messageType) + stmt.executeQuery().use { rs -> + check(rs.next()) { "no row for $messageType" } + Row(OutboxStatus.from(rs.getString("status")), rs.getInt("retries"), rs.getString("last_error")) + } + } + } + } + + test("[$dbName] control: the database rejects the raw error value") { + val entry = seed("poisoned") + shouldThrowAny { + jdbcProvider().withTransaction { + jdbcProvider().withConnection { conn -> + conn.prepareStatement("UPDATE okapi_outbox SET last_error = ? WHERE message_type = ?").use { stmt -> + stmt.setString(1, rejectedError) + stmt.setString(2, entry.messageType) + stmt.executeUpdate() + } + } + } + } + } + + test("[$dbName] the store sanitizes an entry whose lastError was set without retry/toFailed") { + val entry = seed("poisoned") + val unsanitized = entry.copy(status = OutboxStatus.PENDING, retries = 1, lastError = rejectedError) + + jdbcProvider().withTransaction { storeFactory().updateAfterProcessingBatch(listOf(unsanitized)) } + + val stored = checkNotNull(row("poisoned").lastError) + stored.length shouldBeLessThanOrEqual OutboxEntry.MAX_LAST_ERROR_LENGTH + stored shouldNotContain "\u0000" + } + + test("[$dbName] a rejected error value no longer rolls back the delivered entries of its batch") { + seed("healthy") + seed("poisoned") + val processor = OutboxProcessor( + store = storeFactory(), + entryProcessor = OutboxEntryProcessor(SplitDeliverer(rejectedError), RetryPolicy(maxRetries = 5), clock), + ) + + jdbcProvider().withTransaction { processor.processNext(10) } shouldBe 2 + + row("healthy").status shouldBe OutboxStatus.DELIVERED + val poisoned = row("poisoned") + poisoned.status shouldBe OutboxStatus.PENDING + poisoned.retries shouldBe 1 + val stored = checkNotNull(poisoned.lastError) + stored.length shouldBeLessThanOrEqual OutboxEntry.MAX_LAST_ERROR_LENGTH + stored shouldNotContain "\u0000" + } +} diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorPoisonBatchTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorPoisonBatchTest.kt new file mode 100644 index 0000000..c9e7474 --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorPoisonBatchTest.kt @@ -0,0 +1,20 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.mysql.MysqlOutboxStore +import com.softwaremill.okapi.test.support.MysqlTestSupport +import io.kotest.core.spec.style.FunSpec + +class MysqlLastErrorPoisonBatchTest : FunSpec({ + val db = MysqlTestSupport() + + lastErrorPoisonBatchTests( + dbName = "mysql", + jdbcProvider = { db.jdbc }, + storeFactory = { MysqlOutboxStore(db.jdbc) }, + startDb = { db.start() }, + stopDb = { db.stop() }, + truncate = { db.truncate() }, + // `last_error` is TEXT: 65,535 bytes, and strict mode rejects anything longer. + rejectedError = "HTTP 503: " + "x".repeat(100_000), + ) +}) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresLastErrorPoisonBatchTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresLastErrorPoisonBatchTest.kt new file mode 100644 index 0000000..113cf1e --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresLastErrorPoisonBatchTest.kt @@ -0,0 +1,20 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.postgres.PostgresOutboxStore +import com.softwaremill.okapi.test.support.PostgresTestSupport +import io.kotest.core.spec.style.FunSpec + +class PostgresLastErrorPoisonBatchTest : FunSpec({ + val db = PostgresTestSupport() + + lastErrorPoisonBatchTests( + dbName = "postgres", + jdbcProvider = { db.jdbc }, + storeFactory = { PostgresOutboxStore(db.jdbc) }, + startDb = { db.start() }, + stopDb = { db.stop() }, + truncate = { db.truncate() }, + // Postgres text cannot hold NUL; pgjdbc refuses the parameter outright. + rejectedError = "HTTP 500: binary\u0000body", + ) +}) diff --git a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt index 0b097ec..b7713a5 100644 --- a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt +++ b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt @@ -36,7 +36,8 @@ class MysqlOutboxStore( } else { stmt.setNull(9, Types.TIMESTAMP) } - if (entry.lastError != null) stmt.setString(10, entry.lastError) else stmt.setNull(10, Types.VARCHAR) + val lastError = entry.lastError + if (lastError != null) stmt.setString(10, OutboxEntry.sanitizeLastError(lastError)) else stmt.setNull(10, Types.VARCHAR) stmt.setString(11, entry.deliveryMetadata) stmt.executeUpdate() } @@ -205,7 +206,8 @@ class MysqlOutboxStore( } else { setNull(4, Types.TIMESTAMP) } - if (entry.lastError != null) setString(5, entry.lastError) else setNull(5, Types.VARCHAR) + val lastError = entry.lastError + if (lastError != null) setString(5, OutboxEntry.sanitizeLastError(lastError)) else setNull(5, Types.VARCHAR) // CHAR(36) column โ€” MySQL has no native UUID type, so the id travels as a string. setString(6, entry.outboxId.raw.toString()) return this diff --git a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt index 1cfa0d5..4fce105 100644 --- a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt +++ b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt @@ -36,7 +36,8 @@ class PostgresOutboxStore( } else { stmt.setNull(9, Types.TIMESTAMP) } - if (entry.lastError != null) stmt.setString(10, entry.lastError) else stmt.setNull(10, Types.VARCHAR) + val lastError = entry.lastError + if (lastError != null) stmt.setString(10, OutboxEntry.sanitizeLastError(lastError)) else stmt.setNull(10, Types.VARCHAR) stmt.setString(11, entry.deliveryMetadata) stmt.executeUpdate() } @@ -195,7 +196,8 @@ class PostgresOutboxStore( } else { setNull(4, Types.TIMESTAMP) } - if (entry.lastError != null) setString(5, entry.lastError) else setNull(5, Types.VARCHAR) + val lastError = entry.lastError + if (lastError != null) setString(5, OutboxEntry.sanitizeLastError(lastError)) else setNull(5, Types.VARCHAR) setObject(6, entry.outboxId.raw) return this }