Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,25 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with *
automatically. Cross-type claim order is no longer globally oldest first: the processor rotates
the starting type, and any leftover batch capacity goes to the other types alphabetically.
([#116](https://github.com/softwaremill/okapi/issues/116))
- **One unpersistable `lastError` no longer poisons its whole batch.** All results of a batch are
written in one update inside the claim transaction, so a `last_error` value the database rejected
rolled back the entire batch: entries that had already been delivered went back to `PENDING` with
`retries` unchanged and were redelivered on every tick, indefinitely (three times per tick under
Exposed's default retry). Two such values were easy to hit: an HTTP error body over 64 KB
(MySQL `TEXT` overflow, `ERROR 1406`) and a NUL character (refused by Postgres).
`OutboxEntry.retry`/`toFailed` now strip NULs and truncate to `OutboxEntry.MAX_LAST_ERROR_LENGTH`
(4096 chars, never splitting a surrogate pair), so every transport is covered. Both built-in
stores apply the same `OutboxEntry.sanitizeLastError` again when writing, so an entry built with
`copy(lastError = ...)` is covered too; custom stores should call it before writing `last_error`.
(#130)
- **`HttpMessageDeliverer` no longer buffers whole response bodies, or waits on them forever.**
Success bodies are discarded, and only the first `HttpMessageDeliverer.MAX_ERROR_BODY_BYTES`
(2 KB) of an error body is kept for `lastError`. At most `MAX_DRAINED_BODY_BYTES` (64 KB) of any
body is read: up to that the rest is drained so the connection is reused, past it the body is
cancelled. Each exchange, body included, must also finish within `EXCHANGE_TIMEOUT` (30 s) or it
is cancelled as a `RetriableFailure`; the request timeout only covers waiting for the headers.
Previously every response was read fully into heap, and a server dripping its body held delivery,
and with it the scheduler's claim transaction, open indefinitely. (#130)

### Changed

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,15 @@ data class OutboxEntry(
updatedAt = now,
retries = retries + 1,
lastAttempt = now,
lastError = lastError,
lastError = sanitizeLastError(lastError),
)

/** Returns a copy marked as permanently failed. */
fun toFailed(now: Instant, lastError: String): OutboxEntry = copy(
status = OutboxStatus.FAILED,
updatedAt = now,
lastAttempt = now,
lastError = lastError,
lastError = sanitizeLastError(lastError),
)

/** Returns a copy marked as successfully delivered. */
Expand All @@ -47,6 +47,39 @@ data class OutboxEntry(
)

companion object {
/**
* Upper bound, in UTF-16 chars, on a stored [lastError]. A UTF-16 char encodes to at most
* 3 bytes of UTF-8 (a 4-byte code point takes two chars), so the value is at most ~12 KB,
* well inside MySQL's 65,535-byte `TEXT` column.
*/
const val MAX_LAST_ERROR_LENGTH: Int = 4096

/** Appended to a [lastError] (or an HTTP error body) that was cut short. */
const val TRUNCATION_MARKER: String = "... [truncated]"

/**
* Makes [error] safe to persist (issue #130). Every result of a batch is written in one
* update inside the claim transaction, so a single value the database rejects rolls back the
* whole batch -- including entries that were already delivered -- without advancing
* `retries`, and the next tick redelivers all of them, forever. Two such values are cheap to
* produce: an over-long one (an HTTP error page in the response body overflows MySQL's
* `TEXT`) and one containing NUL, which Postgres refuses in text. NULs are dropped and the
* rest is cut to [MAX_LAST_ERROR_LENGTH], never between the two halves of a surrogate pair,
* which MySQL would reject as an invalid string.
*
* [retry] and [toFailed] apply it, and the built-in stores apply it again when writing, so an
* entry built with `copy(lastError = ...)` is covered too. Custom [OutboxStore]s should call
* it before writing `last_error`. It is idempotent.
*/
@JvmStatic
fun sanitizeLastError(error: String): String {
val withoutNul = if (error.indexOf('\u0000') >= 0) error.replace("\u0000", "") else error
if (withoutNul.length <= MAX_LAST_ERROR_LENGTH) return withoutNul
var end = MAX_LAST_ERROR_LENGTH - TRUNCATION_MARKER.length
if (Character.isHighSurrogate(withoutNul[end - 1])) end--
return withoutNul.substring(0, end) + TRUNCATION_MARKER
}

/** Creates a new PENDING entry from a [message] and [deliveryInfo]. */
@JvmStatic
fun createPending(message: OutboxMessage, deliveryInfo: DeliveryInfo, now: Instant): OutboxEntry =
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package com.softwaremill.okapi.core

import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.ints.shouldBeLessThanOrEqual
import io.kotest.matchers.shouldBe
import io.kotest.matchers.string.shouldEndWith
import io.kotest.matchers.string.shouldNotContain
import io.kotest.matchers.string.shouldStartWith
import java.time.Instant

class OutboxEntryLastErrorTest : FunSpec({
val info = object : DeliveryInfo {
override val type = "stub"
override fun serialize(): String = """{"type":"stub"}"""
}
val entry = OutboxEntry.createPending(OutboxMessage("test.event", "{}"), info, Instant.EPOCH)
val now = Instant.parse("2024-01-01T00:00:00Z")

test("a short error is stored unchanged") {
entry.retry(now, "HTTP 503: unavailable").lastError shouldBe "HTTP 503: unavailable"
entry.toFailed(now, "HTTP 400: bad").lastError shouldBe "HTTP 400: bad"
}

test("an over-long error is truncated to MAX_LAST_ERROR_LENGTH with a marker, on retry and on failure") {
val huge = "HTTP 503: " + "x".repeat(100_000)
listOf(entry.retry(now, huge), entry.toFailed(now, huge)).forEach { updated ->
val stored = checkNotNull(updated.lastError)
stored.length shouldBeLessThanOrEqual OutboxEntry.MAX_LAST_ERROR_LENGTH
stored shouldStartWith "HTTP 503: xxx"
stored shouldEndWith OutboxEntry.TRUNCATION_MARKER
}
}

test("an error of exactly MAX_LAST_ERROR_LENGTH is not truncated") {
val exact = "y".repeat(OutboxEntry.MAX_LAST_ERROR_LENGTH)
entry.retry(now, exact).lastError shouldBe exact
}

test("NUL characters are removed") {
entry.retry(now, "binary\u0000body\u0000").lastError shouldBe "binarybody"
}

test("truncation never splits a surrogate pair") {
// Pad so the cut lands right after the high surrogate of an emoji.
val cut = OutboxEntry.MAX_LAST_ERROR_LENGTH - OutboxEntry.TRUNCATION_MARKER.length
val error = "a".repeat(cut - 1) + "😀" + "b".repeat(10_000)
val stored = checkNotNull(entry.retry(now, error).lastError)
stored.toCharArray().forEachIndexed { i, c ->
if (Character.isHighSurrogate(c)) Character.isLowSurrogate(stored.getOrElse(i + 1) { ' ' }) shouldBe true
}
stored shouldNotContain "\uD83D..."
}
})
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,20 @@ import com.softwaremill.okapi.core.DeliveryResult
import com.softwaremill.okapi.core.MessageDeliverer
import com.softwaremill.okapi.core.OutboxEntry
import com.softwaremill.okapi.core.OutboxHeaders
import java.io.ByteArrayOutputStream
import java.io.IOException
import java.net.URI
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.nio.ByteBuffer
import java.time.Duration
import java.util.concurrent.CompletableFuture
import java.util.concurrent.CompletionException
import java.util.concurrent.CompletionStage
import java.util.concurrent.Flow
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
import javax.net.ssl.SSLException

/**
Expand All @@ -37,23 +44,29 @@ import javax.net.ssl.SSLException
* sequentially on [HttpClient.send]. The JDK [HttpClient] will use HTTP/2 multiplexing when
* supported by the server; otherwise it will fall back to HTTP/1.1 (potentially with multiple
* connections) — either way, requests can be overlapped without explicit per-host grouping.
*
* Each exchange, response body included, must complete within [EXCHANGE_TIMEOUT]; otherwise it is
* cancelled and classified as [DeliveryResult.RetriableFailure]. [HttpRequest.timeout] alone only
* covers waiting for the response headers, so a server dripping a body could otherwise hold
* delivery, and with it the scheduler's claim transaction, open indefinitely.
*/
class HttpMessageDeliverer @JvmOverloads constructor(
class HttpMessageDeliverer internal constructor(
private val urlResolver: ServiceUrlResolver,
private val httpClient: HttpClient = defaultHttpClient(),
private val retriableStatusCodes: Set<Int> = DEFAULT_RETRIABLE_STATUS_CODES,
private val httpClient: HttpClient,
private val retriableStatusCodes: Set<Int>,
private val exchangeTimeout: Duration,
) : MessageDeliverer {
@JvmOverloads constructor(
urlResolver: ServiceUrlResolver,
httpClient: HttpClient = defaultHttpClient(),
retriableStatusCodes: Set<Int> = DEFAULT_RETRIABLE_STATUS_CODES,
) : this(urlResolver, httpClient, retriableStatusCodes, EXCHANGE_TIMEOUT)

override val type: String = HttpDeliveryInfo.TYPE

override fun deliver(entry: OutboxEntry): DeliveryResult = try {
val request = buildRequest(entry)
val response = httpClient.send(request, HttpResponse.BodyHandlers.ofString())
classifyResponse(response.statusCode(), response.body())
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
classifyThrowable(e)
} catch (e: Exception) {
classifyThrowable(e)
override fun deliver(entry: OutboxEntry): DeliveryResult = when (val attempt = fireOne(entry)) {
is SendAttempt.ImmediateFailure -> attempt.result
is SendAttempt.InFlight -> awaitResult(attempt)
}

/**
Expand All @@ -70,7 +83,7 @@ class HttpMessageDeliverer @JvmOverloads constructor(
return attempts.map { (entry, attempt) ->
val result = when (attempt) {
is SendAttempt.ImmediateFailure -> attempt.result
is SendAttempt.InFlight -> awaitResult(attempt.future)
is SendAttempt.InFlight -> awaitResult(attempt)
}
DeliveryOutcome(entry, result)
}
Expand All @@ -80,12 +93,14 @@ class HttpMessageDeliverer @JvmOverloads constructor(
* Awaits via [CompletableFuture.get] rather than [CompletableFuture.join] so that an
* interrupt on the caller (processor thread shutdown/backpressure) is observed instead of
* being ignored — the flag is restored here, on the interrupted thread itself, and the
* remaining not-yet-awaited futures in the batch will then also fail fast as retriable.
* remaining not-yet-awaited futures in the batch will then also fail fast as retriable. The
* interrupted exchange itself is cancelled, as [HttpClient.send] would.
*/
private fun awaitResult(future: CompletableFuture<DeliveryResult>): DeliveryResult = try {
future.get()
private fun awaitResult(attempt: SendAttempt.InFlight): DeliveryResult = try {
attempt.result.get()
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
attempt.exchange.cancel(true)
classifyThrowable(e)
} catch (e: Exception) {
// fireOne()'s .exceptionally already converts every failure into a normal completion, so
Expand All @@ -94,13 +109,25 @@ class HttpMessageDeliverer @JvmOverloads constructor(
classifyThrowable(e.cause ?: e)
}

/**
* The deadline runs on a copy of the exchange future: completing that copy with a
* [TimeoutException] does not abort the HTTP exchange, so the exchange is cancelled explicitly.
*/
private fun fireOne(entry: OutboxEntry): SendAttempt = try {
val request = buildRequest(entry)
val future: CompletableFuture<DeliveryResult> = httpClient
.sendAsync(request, HttpResponse.BodyHandlers.ofString())
.thenApply { response -> classifyResponse(response.statusCode(), response.body()) }
.exceptionally { e -> classifyThrowable(e.cause ?: e) }
SendAttempt.InFlight(future)
val exchange = httpClient.sendAsync(request, boundedBodyHandler)
val result: CompletableFuture<DeliveryResult> = exchange.copy()
.orTimeout(exchangeTimeout.toMillis(), TimeUnit.MILLISECONDS)
.handle { response, failure ->
if (failure == null) {
classifyResponse(response.statusCode(), response.body())
} else {
val cause = if (failure is CompletionException) failure.cause ?: failure else failure
if (cause is TimeoutException) exchange.cancel(true)
classifyThrowable(cause)
}
}
SendAttempt.InFlight(exchange, result)
} catch (e: Exception) {
SendAttempt.ImmediateFailure(classifyThrowable(e))
}
Expand Down Expand Up @@ -129,6 +156,18 @@ class HttpMessageDeliverer @JvmOverloads constructor(
.build()
}

/**
* Discards success bodies and keeps at most [MAX_ERROR_BODY_BYTES] of an error body (issue #130).
* The body only ever feeds `lastError`, and `BodyHandlers.ofString()` buffered all of it in heap
* -- an endpoint answering with a multi-megabyte error page cost that much memory per in-flight
* request, and the full text then overflowed the `last_error` column. Either way at most
* [MAX_DRAINED_BODY_BYTES] are read; see [BoundedUtf8BodySubscriber].
*/
private val boundedBodyHandler = HttpResponse.BodyHandler { info ->
val keep = if (info.statusCode() in 200..299) 0 else MAX_ERROR_BODY_BYTES
BoundedUtf8BodySubscriber(keep, MAX_DRAINED_BODY_BYTES)
}

private fun classifyResponse(status: Int, body: String): DeliveryResult = when {
status in 200..299 -> DeliveryResult.Success
status in retriableStatusCodes -> DeliveryResult.RetriableFailure("HTTP $status: $body")
Expand All @@ -142,6 +181,7 @@ class HttpMessageDeliverer @JvmOverloads constructor(
is JsonProcessingException -> DeliveryResult.PermanentFailure(message)
is SSLException -> DeliveryResult.PermanentFailure(message)
is IOException -> DeliveryResult.RetriableFailure(message)
is TimeoutException -> DeliveryResult.RetriableFailure("HTTP exchange did not complete within $exchangeTimeout")
// Deliberately side-effect free: this can run on an HttpClient completion thread
// (via fireOne()'s .exceptionally), not the caller's thread, so interrupting
// "current thread" here would interrupt an unrelated shared pool thread. Callers on
Expand All @@ -152,11 +192,25 @@ class HttpMessageDeliverer @JvmOverloads constructor(
}

private sealed interface SendAttempt {
data class InFlight(val future: CompletableFuture<DeliveryResult>) : SendAttempt
/** [exchange] is the raw [HttpClient.sendAsync] future, kept so the exchange can be cancelled. */
data class InFlight(val exchange: CompletableFuture<*>, val result: CompletableFuture<DeliveryResult>) : SendAttempt
data class ImmediateFailure(val result: DeliveryResult) : SendAttempt
}

companion object {
/** How much of an error response body is kept for `lastError`; the rest is dropped. */
const val MAX_ERROR_BODY_BYTES: Int = 2048

/**
* How much of any response body is read. Up to this size the body is drained so the
* connection can be reused; past it the body is cancelled, which closes an HTTP/1.1
* connection (HTTP/2 resets only the stream).
*/
const val MAX_DRAINED_BODY_BYTES: Long = 64 * 1024

/** Upper bound on one exchange, from sending the request to the end of the response body. */
val EXCHANGE_TIMEOUT: Duration = Duration.ofSeconds(30)

val DEFAULT_RETRIABLE_STATUS_CODES: Set<Int> = (500..599).toSet() + 429 + 408
private val CONNECT_TIMEOUT: Duration = Duration.ofSeconds(10)
private val REQUEST_TIMEOUT: Duration = Duration.ofSeconds(30)
Expand All @@ -167,3 +221,59 @@ class HttpMessageDeliverer @JvmOverloads constructor(
.build()
}
}

/**
* Keeps the first [keepLimit] bytes of a body as UTF-8 and reads at most [readLimit] bytes in total.
* Below [readLimit] the rest is drained without being stored, so the connection stays reusable;
* past it the subscription is cancelled rather than reading an arbitrarily large or endless body.
* A multi-byte character cut at [keepLimit] decodes as U+FFFD, which is fine for an error message.
*
* The HTTP client signals this subscriber serially (Reactive Streams §1.3), so its state needs no
* synchronization.
*/
private class BoundedUtf8BodySubscriber(
private val keepLimit: Int,
private val readLimit: Long,
) : HttpResponse.BodySubscriber<String> {
private val body = CompletableFuture<String>()
private val kept = ByteArrayOutputStream()
private var read = 0L
private var done = false
private lateinit var subscription: Flow.Subscription

override fun getBody(): CompletionStage<String> = body

override fun onSubscribe(subscription: Flow.Subscription) {
this.subscription = subscription
subscription.request(Long.MAX_VALUE)
}

override fun onNext(item: List<ByteBuffer>) {
if (done) return
for (buffer in item) {
val take = minOf(keepLimit - kept.size(), buffer.remaining())
if (take > 0) kept.write(ByteArray(take).also { buffer.get(it) })
read += take + buffer.remaining()
}
if (read > readLimit) {
subscription.cancel()
finish()
}
}

override fun onError(throwable: Throwable) {
if (done) return
done = true
body.completeExceptionally(throwable)
}

override fun onComplete() {
if (!done) finish()
}

private fun finish() {
done = true
val truncated = read > kept.size()
body.complete(kept.toString(Charsets.UTF_8) + if (truncated && keepLimit > 0) OutboxEntry.TRUNCATION_MARKER else "")
}
}
Loading