From 8936b97e24f66ee07e14d99df2c2cda6af563b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Wokacz?= Date: Thu, 12 Nov 2026 12:35:14 +0100 Subject: [PATCH 1/2] fix(stores): isolate a rejected row so one last_error cannot roll back a batch; MySQL last_error is utf8mb4 (#141) When the batch status update fails, PostgresOutboxStore and MysqlOutboxStore roll back to a savepoint and write the rows one by one, each under its own savepoint. A row the database still rejects is written with an ASCII placeholder last_error, keeping its status and retries; the original message is logged at WARN. A row rejected for another reason still fails the batch. MySQL changeset 005 converts last_error to utf8mb4, so a latin1/utf8mb3 schema no longer rejects emoji and other characters outside its charset. Its precondition skips the table rebuild where the column is already utf8mb4. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 16 ++ README.md | 2 +- .../softwaremill/okapi/core/OutboxEntry.kt | 4 + .../softwaremill/okapi/core/OutboxStore.kt | 5 + .../test/store/MysqlLastErrorCharsetTest.kt | 166 ++++++++++++++++++ .../store/MysqlRejectedRowIsolationTest.kt | 36 ++++ .../store/PostgresRejectedRowIsolationTest.kt | 41 +++++ .../test/store/RejectedRowIsolationTests.kt | 124 +++++++++++++ okapi-mysql/build.gradle.kts | 1 + .../okapi/mysql/MysqlOutboxStore.kt | 71 +++++++- .../db/mysql/005__last_error_utf8mb4.sql | 19 ++ .../softwaremill/okapi/db/mysql/changelog.xml | 1 + okapi-postgres/build.gradle.kts | 1 + .../okapi/postgres/PostgresOutboxStore.kt | 71 +++++++- 14 files changed, 549 insertions(+), 9 deletions(-) create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorCharsetTest.kt create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlRejectedRowIsolationTest.kt create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresRejectedRowIsolationTest.kt create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/RejectedRowIsolationTests.kt create mode 100644 okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f00b063..8b850290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,22 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * they try again on their next one. A runner call inside an existing transaction, and every other Exposed transaction in the application, keep their configured retries. ([#134](https://github.com/softwaremill/okapi/issues/134)) +- **One rejected `last_error` no longer rolls back a whole batch, whatever the cause.** #130 removed the + two known values the database rejected, but any other value still made the batch update fail, rolling + back the batch's results, delivered entries included, which were then delivered again on every tick. + When the batch update fails, `PostgresOutboxStore` and `MysqlOutboxStore` now roll back to a savepoint + and write the rows one by one, each under its own savepoint. A row the database still rejects is + written with a placeholder `last_error`, keeping its status and `retries`, and the original message is + logged at WARN with the entry id. A row rejected for a reason other than its error message still fails + the batch, as before. A successful batch costs two extra statements, `SAVEPOINT` and `RELEASE SAVEPOINT`. + ([#141](https://github.com/softwaremill/okapi/issues/141)) +- **MySQL: `last_error` is `utf8mb4` (new migration `005__last_error_utf8mb4`).** 001 created + `okapi_outbox` in the schema's default character set. On a `latin1` or `utf8mb3` schema, common on + servers upgraded from MySQL 5.7, an error message with an emoji or any other character outside that + set was rejected with `Incorrect string value`. 005 converts the column to `utf8mb4`. That rebuilds + the table, and writes, `publish()` included, wait until it finishes, so on a large outbox run it in a + quiet period. Its precondition skips it where the column is already `utf8mb4`, the MySQL 8 default. + ([#141](https://github.com/softwaremill/okapi/issues/141)) - **MySQL: the processor no longer blocks `publish()` for the duration of delivery.** Under MySQL's default `REPEATABLE READ`, the claim's `FOR UPDATE SKIP LOCKED` took gap locks on the claim index; with a backlog smaller than the batch, the gap ran to the end of the claimed type's range, exactly diff --git a/README.md b/README.md index a09be970..05a382a0 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ With `okapi-spring-boot` and Liquibase on the classpath, these run automatically If you use another migration tool, apply every SQL migration for your database, in order: - **PostgreSQL:** [001](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql) route-aware claim index, [003](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql) `TIMESTAMPTZ` timestamp columns, [004](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/004__retry_backoff.sql) retry backoff column and claim index (its second changeset must run outside a transaction) -- **MySQL:** [001](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql) route-aware claim index, [003](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql) `DATETIME(6)` timestamp columns, [004](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/004__retry_backoff.sql) retry backoff column and claim index +- **MySQL:** [001](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql) route-aware claim index, [003](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql) `DATETIME(6)` timestamp columns, [004](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/004__retry_backoff.sql) retry backoff column and claim index, [005](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql) `utf8mb4` `last_error` (a no-op where it already is) For an existing installation, apply every new migration (up to `004`) before starting updated workers. MySQL claims explicitly select the claim index from `004` and fail if it is missing. PostgreSQL's `002` and the index changeset of `004` use `CREATE INDEX CONCURRENTLY`, so run them outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. `CREATE INDEX CONCURRENTLY` waits for every transaction in the database that started before it, not only those on `okapi_outbox`, so long-running transactions (including old workers delivering a claimed batch) delay it, and with Liquibase they delay application startup while the changelog lock is held. Run it when no long transactions are open, or apply it ahead of the rollout. diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntry.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntry.kt index 2e7a6ec2..a9074d7a 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntry.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxEntry.kt @@ -77,6 +77,10 @@ data class OutboxEntry @JvmOverloads constructor( * rest is cut to [MAX_LAST_ERROR_LENGTH], never between the two halves of a surrogate pair, * which MySQL would reject as an invalid string. * + * The built-in stores also survive a value rejected for any other reason, by writing a + * placeholder in its place (issue #141). Sanitizing still matters, because the placeholder + * loses the message. + * * [retry] and [toFailed] apply it, and the built-in stores apply it again when writing, so an * entry built with `copy(lastError = ...)` is covered too. Custom [OutboxStore]s should call * it before writing `last_error`. It is idempotent. diff --git a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxStore.kt b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxStore.kt index a5ada0ed..3e32eebc 100644 --- a/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxStore.kt +++ b/okapi-core/src/main/kotlin/com/softwaremill/okapi/core/OutboxStore.kt @@ -39,6 +39,11 @@ interface OutboxStore { * Updates a batch of entries after a delivery attempt in a single roundtrip where * possible. Default loops over [updateAfterProcessing] one at a time; storage modules * override this with a single JDBC `executeBatch()` call. + * + * It runs in the claim's transaction, so if it throws, the results of the whole batch roll + * back, including entries already delivered, and they are delivered again. The built-in stores + * therefore write the rows one by one when the batch update fails, and store a placeholder + * `last_error` for a row the database still rejects (issue #141). */ fun updateAfterProcessingBatch(entries: List): List = entries.map { updateAfterProcessing(it) } } diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorCharsetTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorCharsetTest.kt new file mode 100644 index 00000000..81c63b04 --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorCharsetTest.kt @@ -0,0 +1,166 @@ +package com.softwaremill.okapi.test.store + +import com.mysql.cj.jdbc.MysqlDataSource +import com.softwaremill.okapi.core.DeliveryInfo +import com.softwaremill.okapi.core.DeliveryResult +import com.softwaremill.okapi.core.MessageDeliverer +import com.softwaremill.okapi.core.OutboxEntry +import com.softwaremill.okapi.core.OutboxEntryProcessor +import com.softwaremill.okapi.core.OutboxMessage +import com.softwaremill.okapi.core.OutboxProcessor +import com.softwaremill.okapi.core.OutboxStatus +import com.softwaremill.okapi.core.RetryPolicy +import com.softwaremill.okapi.mysql.MysqlOutboxStore +import com.softwaremill.okapi.test.support.JdbcConnectionProvider +import io.kotest.assertions.throwables.shouldThrowAny +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldStartWith +import liquibase.Liquibase +import liquibase.database.DatabaseFactory +import liquibase.database.jvm.JdbcConnection +import liquibase.resource.ClassLoaderResourceAccessor +import org.testcontainers.containers.MySQLContainer +import java.sql.Connection +import java.sql.DriverManager +import java.time.Clock + +private const val CHANGELOG = "com/softwaremill/okapi/db/mysql/changelog.xml" + +/** Changesets in 001 to 004, i.e. everything before 005. */ +private const val CHANGESETS_BEFORE_005 = 7 + +private const val EMOJI_ERROR = "HTTP 503: upstream is on fire 🔥" + +private object CharsetTestDeliveryInfo : DeliveryInfo { + override val type: String = "charset-test" + override fun serialize(): String = """{"type":"charset-test"}""" +} + +private object EmojiErrorDeliverer : MessageDeliverer { + override val type: String = CharsetTestDeliveryInfo.type + override fun deliver(entry: OutboxEntry): DeliveryResult = when (entry.messageType) { + "healthy" -> DeliveryResult.Success + else -> DeliveryResult.RetriableFailure(EMOJI_ERROR) + } +} + +/** + * Issue #141 on a MySQL schema whose default character set is latin1, as on servers upgraded from + * 5.7. 001 creates `okapi_outbox` in the schema's character set, where an emoji in `last_error` is + * rejected; changeset 005 converts the column to utf8mb4. + */ +class MysqlLastErrorCharsetTest : FunSpec({ + val container = MySQLContainer("mysql:8.0") + val clock = Clock.systemUTC() + + beforeSpec { + container.start() + DriverManager.getConnection(container.jdbcUrl, "root", container.password).use { conn -> + conn.createStatement().use { + it.execute("CREATE DATABASE legacy CHARACTER SET latin1") + it.execute("CREATE DATABASE modern") + } + } + } + afterSpec { container.stop() } + + fun url(database: String) = container.jdbcUrl.replaceAfterLast('/', database) + + fun connect(database: String): Connection = DriverManager.getConnection(url(database), "root", container.password) + + /** Runs the first [changes] changesets (all of them when null). */ + fun migrate(database: String, changes: Int? = null) { + connect(database).use { conn -> + Liquibase( + CHANGELOG, + ClassLoaderResourceAccessor(), + DatabaseFactory.getInstance().findCorrectDatabaseImplementation(JdbcConnection(conn)), + ).use { if (changes != null) it.update(changes, "") else it.update("") } + } + } + + fun jdbc(database: String) = JdbcConnectionProvider( + MysqlDataSource().apply { + setURL(url(database)) + user = "root" + setPassword(container.password) + }, + ) + + fun query(database: String, sql: String): String? = connect(database).use { conn -> + conn.createStatement().use { stmt -> stmt.executeQuery(sql).use { rs -> if (rs.next()) rs.getString(1) else null } } + } + + fun lastErrorCharset(database: String) = query( + database, + "SELECT CHARACTER_SET_NAME FROM information_schema.COLUMNS " + + "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'okapi_outbox' AND COLUMN_NAME = 'last_error'", + ) + + fun execType005(database: String) = query(database, "SELECT EXECTYPE FROM DATABASECHANGELOG WHERE ID = '005'") + + /** Seeds a `healthy` and a `poisoned` entry, processes them in one batch and returns the stored rows. */ + fun processHealthyAndPoisoned(database: String): Map> { + connect(database).use { conn -> conn.createStatement().use { it.execute("DELETE FROM okapi_outbox") } } + val jdbc = jdbc(database) + val store = MysqlOutboxStore(jdbc) + jdbc.withTransaction { + listOf("healthy", "poisoned").forEach { + store.persist(OutboxEntry.createPending(OutboxMessage(it, "{}"), CharsetTestDeliveryInfo, clock.instant())) + } + } + val processor = OutboxProcessor(store, OutboxEntryProcessor(EmojiErrorDeliverer, RetryPolicy(maxRetries = 5), clock)) + jdbc.withTransaction { processor.processNext(10) } shouldBe 2 + + return connect(database).use { conn -> + conn.createStatement().use { stmt -> + stmt.executeQuery("SELECT message_type, status, last_error FROM okapi_outbox").use { rs -> + generateSequence { + if (rs.next()) rs.getString(1) to (OutboxStatus.from(rs.getString(2)) to rs.getString(3)) else null + }.toMap() + } + } + } + } + + test("before 005, a latin1 last_error rejects an emoji, and the store keeps the batch with a placeholder error") { + migrate("legacy", CHANGESETS_BEFORE_005) + lastErrorCharset("legacy") shouldBe "latin1" + // Control: the raw value really is rejected. + shouldThrowAny { + connect("legacy").use { conn -> + conn.prepareStatement( + "INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, delivery_metadata, last_error) " + + "VALUES (UUID(), 'm', '{}', 'charset-test', '{}', ?)", + ).use { + it.setString(1, EMOJI_ERROR) + it.executeUpdate() + } + } + } + + val rows = processHealthyAndPoisoned("legacy") + + rows.getValue("healthy").first shouldBe OutboxStatus.DELIVERED + rows.getValue("poisoned").first shouldBe OutboxStatus.PENDING + checkNotNull(rows.getValue("poisoned").second) shouldStartWith "okapi: the database rejected this entry's error message" + } + + test("005 converts last_error to utf8mb4, which stores the emoji") { + migrate("legacy") + execType005("legacy") shouldBe "EXECUTED" + lastErrorCharset("legacy") shouldBe "utf8mb4" + + val rows = processHealthyAndPoisoned("legacy") + + rows.getValue("healthy").first shouldBe OutboxStatus.DELIVERED + rows.getValue("poisoned") shouldBe (OutboxStatus.PENDING to EMOJI_ERROR) + } + + test("005 does nothing where last_error is already utf8mb4") { + migrate("modern") + lastErrorCharset("modern") shouldBe "utf8mb4" + execType005("modern") shouldBe "MARK_RAN" + } +}) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlRejectedRowIsolationTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlRejectedRowIsolationTest.kt new file mode 100644 index 00000000..0851d0fd --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlRejectedRowIsolationTest.kt @@ -0,0 +1,36 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.mysql.MysqlOutboxStore +import com.softwaremill.okapi.test.support.MysqlTestSupport +import io.kotest.core.spec.style.FunSpec +import java.sql.DriverManager + +class MysqlRejectedRowIsolationTest : FunSpec({ + val db = MysqlTestSupport() + + rejectedRowIsolationTests( + dbName = "mysql", + jdbcProvider = { db.jdbc }, + storeFactory = { MysqlOutboxStore(db.jdbc) }, + startDb = { db.start() }, + stopDb = { db.stop() }, + truncate = { db.truncate() }, + installRejectingTrigger = { + // With binary logging on, creating a trigger takes SUPER, which only root has. + DriverManager.getConnection(db.container.jdbcUrl, "root", db.container.password).use { conn -> + conn.createStatement().use { + it.execute( + """ + CREATE TRIGGER okapi_test_reject BEFORE UPDATE ON okapi_outbox FOR EACH ROW + BEGIN + IF NEW.last_error LIKE '%$REJECTED_ERROR_MARKER%' OR NEW.message_type = '$ALWAYS_REJECTED_TYPE' THEN + SIGNAL SQLSTATE '22000' SET MESSAGE_TEXT = 'rejected by test trigger'; + END IF; + END + """.trimIndent(), + ) + } + } + }, + ) +}) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresRejectedRowIsolationTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresRejectedRowIsolationTest.kt new file mode 100644 index 00000000..041c153d --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresRejectedRowIsolationTest.kt @@ -0,0 +1,41 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.postgres.PostgresOutboxStore +import com.softwaremill.okapi.test.support.PostgresTestSupport +import io.kotest.core.spec.style.FunSpec + +class PostgresRejectedRowIsolationTest : FunSpec({ + val db = PostgresTestSupport() + + rejectedRowIsolationTests( + dbName = "postgres", + jdbcProvider = { db.jdbc }, + storeFactory = { PostgresOutboxStore(db.jdbc) }, + startDb = { db.start() }, + stopDb = { db.stop() }, + truncate = { db.truncate() }, + installRejectingTrigger = { + db.jdbc.withTransaction { + db.jdbc.withConnection { conn -> + conn.createStatement().use { + it.execute( + """ + CREATE FUNCTION okapi_test_reject() RETURNS trigger AS ${'$'}${'$'} + BEGIN + IF NEW.last_error LIKE '%$REJECTED_ERROR_MARKER%' OR NEW.message_type = '$ALWAYS_REJECTED_TYPE' THEN + RAISE EXCEPTION 'rejected by test trigger' USING ERRCODE = '22000'; + END IF; + RETURN NEW; + END ${'$'}${'$'} LANGUAGE plpgsql + """.trimIndent(), + ) + it.execute( + "CREATE TRIGGER okapi_test_reject BEFORE UPDATE ON okapi_outbox " + + "FOR EACH ROW EXECUTE FUNCTION okapi_test_reject()", + ) + } + } + } + }, + ) +}) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/RejectedRowIsolationTests.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/RejectedRowIsolationTests.kt new file mode 100644 index 00000000..a40e9612 --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/RejectedRowIsolationTests.kt @@ -0,0 +1,124 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.core.DeliveryInfo +import com.softwaremill.okapi.core.DeliveryResult +import com.softwaremill.okapi.core.MessageDeliverer +import com.softwaremill.okapi.core.OutboxEntry +import com.softwaremill.okapi.core.OutboxEntryProcessor +import com.softwaremill.okapi.core.OutboxMessage +import com.softwaremill.okapi.core.OutboxProcessor +import com.softwaremill.okapi.core.OutboxStatus +import com.softwaremill.okapi.core.OutboxStore +import com.softwaremill.okapi.core.RetryPolicy +import com.softwaremill.okapi.test.support.JdbcConnectionProvider +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldStartWith +import java.sql.SQLException +import java.time.Clock + +/** Marker in a `last_error` that the test trigger rejects. */ +const val REJECTED_ERROR_MARKER = "REJECT_ME" + +/** `message_type` whose rows the test trigger rejects on every update. */ +const val ALWAYS_REJECTED_TYPE = "always-rejected" + +private object IsolationTestDeliveryInfo : DeliveryInfo { + override val type: String = "isolation-test" + override fun serialize(): String = """{"type":"isolation-test"}""" +} + +/** Delivers `healthy` entries and fails all others with a retriable error carrying the trigger's marker. */ +private object HealthyOnlyDeliverer : MessageDeliverer { + override val type: String = IsolationTestDeliveryInfo.type + override fun deliver(entry: OutboxEntry): DeliveryResult = when { + entry.messageType.startsWith("healthy") -> DeliveryResult.Success + else -> DeliveryResult.RetriableFailure("HTTP 503 $REJECTED_ERROR_MARKER") + } +} + +private data class IsolationRow(val status: OutboxStatus, val retries: Int, val lastError: String?) + +/** + * Issue #141: a row whose update the database rejects for any reason must not roll back the results + * of the rest of its batch. [installRejectingTrigger] adds a BEFORE UPDATE trigger that rejects an + * update whose `last_error` contains [REJECTED_ERROR_MARKER], or of a row whose `message_type` is + * [ALWAYS_REJECTED_TYPE], standing in for whatever the database might refuse. + */ +fun FunSpec.rejectedRowIsolationTests( + dbName: String, + jdbcProvider: () -> JdbcConnectionProvider, + storeFactory: () -> OutboxStore, + startDb: () -> Unit, + stopDb: () -> Unit, + truncate: () -> Unit, + installRejectingTrigger: () -> Unit, +) { + val clock = Clock.systemUTC() + + beforeSpec { + startDb() + installRejectingTrigger() + } + afterSpec { stopDb() } + beforeEach { truncate() } + + fun seed(messageType: String) { + jdbcProvider().withTransaction { + storeFactory().persist(OutboxEntry.createPending(OutboxMessage(messageType, "{}"), IsolationTestDeliveryInfo, clock.instant())) + } + } + + fun row(messageType: String): IsolationRow = jdbcProvider().withTransaction { + jdbcProvider().withConnection { conn -> + conn.prepareStatement("SELECT status, retries, last_error FROM okapi_outbox WHERE message_type = ?").use { stmt -> + stmt.setString(1, messageType) + stmt.executeQuery().use { rs -> + check(rs.next()) { "no row for $messageType" } + IsolationRow(OutboxStatus.from(rs.getString("status")), rs.getInt("retries"), rs.getString("last_error")) + } + } + } + } + + fun processor() = OutboxProcessor( + store = storeFactory(), + entryProcessor = OutboxEntryProcessor(HealthyOnlyDeliverer, RetryPolicy(maxRetries = 5), clock), + ) + + test("[$dbName] a row whose last_error is rejected keeps its status and retries, and the rest of the batch commits") { + seed("healthy") + seed("poisoned") + seed("healthy-2") + + jdbcProvider().withTransaction { processor().processNext(10) } shouldBe 3 + + row("healthy").status shouldBe OutboxStatus.DELIVERED + row("healthy-2").status shouldBe OutboxStatus.DELIVERED + val poisoned = row("poisoned") + poisoned.status shouldBe OutboxStatus.PENDING + poisoned.retries shouldBe 1 + checkNotNull(poisoned.lastError) shouldStartWith "okapi: the database rejected this entry's error message" + } + + test("[$dbName] a row rejected whatever its last_error still fails the batch, and nothing is written") { + seed("healthy") + seed(ALWAYS_REJECTED_TYPE) + + shouldThrow { jdbcProvider().withTransaction { processor().processNext(10) } } + + row("healthy").status shouldBe OutboxStatus.PENDING + row(ALWAYS_REJECTED_TYPE).retries shouldBe 0 + } + + test("[$dbName] a batch with no rejected row is written as before") { + seed("healthy") + seed("healthy-2") + + jdbcProvider().withTransaction { processor().processNext(10) } shouldBe 2 + + row("healthy") shouldBe IsolationRow(OutboxStatus.DELIVERED, 0, null) + row("healthy-2") shouldBe IsolationRow(OutboxStatus.DELIVERED, 0, null) + } +} diff --git a/okapi-mysql/build.gradle.kts b/okapi-mysql/build.gradle.kts index dd671e56..b817486c 100644 --- a/okapi-mysql/build.gradle.kts +++ b/okapi-mysql/build.gradle.kts @@ -7,6 +7,7 @@ description = "MySQL outbox store using plain JDBC" dependencies { implementation(project(":okapi-core")) + implementation(libs.slf4jApi) compileOnly(libs.liquibaseCore) diff --git a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt index 24878dd4..a32bca57 100644 --- a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt +++ b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt @@ -5,8 +5,11 @@ import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxId import com.softwaremill.okapi.core.OutboxStatus import com.softwaremill.okapi.core.RouteAwareOutboxStore +import org.slf4j.LoggerFactory +import java.sql.Connection import java.sql.PreparedStatement import java.sql.ResultSet +import java.sql.SQLException import java.sql.Types import java.time.Instant import java.time.LocalDateTime @@ -170,16 +173,70 @@ class MysqlOutboxStore( override fun updateAfterProcessingBatch(entries: List): List { if (entries.isEmpty()) return entries connectionProvider.withConnection { conn -> - conn.prepareStatement(updateSql).use { stmt -> - entries.forEach { entry -> - stmt.prepareForUpdateFrom(entry).addBatch() + // A row the database rejects must not roll back the results of the rest of the batch (issue #141), + // so on failure the rows are written one by one. + val batchFailure = conn.isolated { + conn.prepareStatement(updateSql).use { stmt -> + entries.forEach { entry -> + stmt.prepareForUpdateFrom(entry).addBatch() + } + stmt.executeBatch() } - stmt.executeBatch() + } + if (batchFailure != null) { + logger.debug("Batch update of {} outbox entries failed; writing them one by one", entries.size, batchFailure) + entries.forEach { conn.updateRowByRow(it) } } } return entries } + /** + * Writes [entry] on its own. If the database rejects it and it has a `last_error`, the most likely + * culprit, it is written again with a placeholder error, so that its status and retries are kept. + */ + private fun Connection.updateRowByRow(entry: OutboxEntry) { + val rejected = isolated { prepareStatement(updateSql).use { it.prepareForUpdateFrom(entry).executeUpdate() } } ?: return + val lastError = entry.lastError ?: throw rejected + logger.warn( + "The database rejected the last_error of outbox entry {}; storing a placeholder. Rejected error (truncated): {}", + entry.outboxId.raw, + lastError.take(LOGGED_ERROR_LENGTH), + rejected, + ) + val placeholder = entry.copy(lastError = "$REJECTED_LAST_ERROR (SQLState ${rejected.sqlState})") + isolated { prepareStatement(updateSql).use { it.prepareForUpdateFrom(placeholder).executeUpdate() } }?.let { + it.addSuppressed(rejected) + throw it + } + } + + /** + * Runs [block] under a savepoint and returns the [SQLException] it threw, after rolling back to the + * savepoint, or `null`. A Postgres transaction accepts no statement after an error until then. When + * the database has already rolled back the whole transaction, e.g. a MySQL deadlock victim, the + * rollback to the savepoint fails and that exception propagates, so nothing is written outside + * the claim's transaction. Without a transaction there is nothing to protect, and no savepoint. + */ + private fun Connection.isolated(block: () -> Unit): SQLException? { + val savepoint = if (autoCommit) null else setSavepoint() + return try { + block() + if (savepoint != null) releaseSavepoint(savepoint) + null + } catch (e: SQLException) { + if (savepoint != null) { + try { + rollback(savepoint) + } catch (rollbackFailure: SQLException) { + rollbackFailure.addSuppressed(e) + throw rollbackFailure + } + } + e + } + } + private fun ResultSet.toOutboxEntry(): OutboxEntry = OutboxEntry( outboxId = OutboxId(UUID.fromString(getString("id"))), messageType = getString("message_type"), @@ -219,6 +276,12 @@ class MysqlOutboxStore( private fun ResultSet.getUtc(column: String): Instant? = getObject(column, LocalDateTime::class.java)?.toInstant(ZoneOffset.UTC) companion object { + private val logger = LoggerFactory.getLogger(MysqlOutboxStore::class.java) + + /** Stored instead of a `last_error` the database rejected. ASCII, so that any charset accepts it. */ + private const val REJECTED_LAST_ERROR = "okapi: the database rejected this entry's error message; see the application log" + private const val LOGGED_ERROR_LENGTH = 2_000 + private val upsertSql = """ INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, last_error, delivery_metadata, next_attempt_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) diff --git a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql new file mode 100644 index 00000000..36f3e9c3 --- /dev/null +++ b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql @@ -0,0 +1,19 @@ +--liquibase formatted sql + +-- last_error holds transport and exception messages, which can contain any Unicode character, such as +-- an emoji in an HTTP error body (issue #141). 001 created the table without a character set, so it +-- took the schema's default. On a latin1 or utf8mb3 schema such a value fails in strict mode with +-- "Incorrect string value". The store then writes a placeholder instead, so the batch survives, but the +-- message is lost. utf8mb4 stores every character. +-- +-- Changing a column's character set rebuilds the table, and writes, publish() included, wait until it +-- finishes. The precondition skips the rebuild where last_error is already utf8mb4: the MySQL 8 default, +-- so on most installations this does nothing. + +--changeset outbox:005 +--preconditions onFail:MARK_RAN +--precondition-sql-check expectedResult:0 SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'okapi_outbox' AND COLUMN_NAME = 'last_error' AND CHARACTER_SET_NAME = 'utf8mb4' +SET @okapi_previous_lock_wait_timeout = @@session.lock_wait_timeout; +SET SESSION lock_wait_timeout = 30; +ALTER TABLE okapi_outbox MODIFY last_error TEXT CHARACTER SET utf8mb4 NULL; +SET SESSION lock_wait_timeout = @okapi_previous_lock_wait_timeout; diff --git a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml index 16bb26c5..1b3e54f1 100644 --- a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml +++ b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml @@ -7,4 +7,5 @@ + diff --git a/okapi-postgres/build.gradle.kts b/okapi-postgres/build.gradle.kts index 6185570d..335e5daf 100644 --- a/okapi-postgres/build.gradle.kts +++ b/okapi-postgres/build.gradle.kts @@ -7,6 +7,7 @@ description = "PostgreSQL outbox store using plain JDBC" dependencies { implementation(project(":okapi-core")) + implementation(libs.slf4jApi) compileOnly(libs.liquibaseCore) diff --git a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt index e54f85c1..8ba37e61 100644 --- a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt +++ b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt @@ -5,8 +5,11 @@ import com.softwaremill.okapi.core.OutboxEntry import com.softwaremill.okapi.core.OutboxId import com.softwaremill.okapi.core.OutboxStatus import com.softwaremill.okapi.core.RouteAwareOutboxStore +import org.slf4j.LoggerFactory +import java.sql.Connection import java.sql.PreparedStatement import java.sql.ResultSet +import java.sql.SQLException import java.sql.Types import java.time.Instant import java.time.OffsetDateTime @@ -158,16 +161,70 @@ class PostgresOutboxStore( override fun updateAfterProcessingBatch(entries: List): List { if (entries.isEmpty()) return entries connectionProvider.withConnection { conn -> - conn.prepareStatement(updateSql).use { stmt -> - entries.forEach { entry -> - stmt.prepareForUpdateFrom(entry).addBatch() + // A row the database rejects must not roll back the results of the rest of the batch (issue #141), + // so on failure the rows are written one by one. + val batchFailure = conn.isolated { + conn.prepareStatement(updateSql).use { stmt -> + entries.forEach { entry -> + stmt.prepareForUpdateFrom(entry).addBatch() + } + stmt.executeBatch() } - stmt.executeBatch() + } + if (batchFailure != null) { + logger.debug("Batch update of {} outbox entries failed; writing them one by one", entries.size, batchFailure) + entries.forEach { conn.updateRowByRow(it) } } } return entries } + /** + * Writes [entry] on its own. If the database rejects it and it has a `last_error`, the most likely + * culprit, it is written again with a placeholder error, so that its status and retries are kept. + */ + private fun Connection.updateRowByRow(entry: OutboxEntry) { + val rejected = isolated { prepareStatement(updateSql).use { it.prepareForUpdateFrom(entry).executeUpdate() } } ?: return + val lastError = entry.lastError ?: throw rejected + logger.warn( + "The database rejected the last_error of outbox entry {}; storing a placeholder. Rejected error (truncated): {}", + entry.outboxId.raw, + lastError.take(LOGGED_ERROR_LENGTH), + rejected, + ) + val placeholder = entry.copy(lastError = "$REJECTED_LAST_ERROR (SQLState ${rejected.sqlState})") + isolated { prepareStatement(updateSql).use { it.prepareForUpdateFrom(placeholder).executeUpdate() } }?.let { + it.addSuppressed(rejected) + throw it + } + } + + /** + * Runs [block] under a savepoint and returns the [SQLException] it threw, after rolling back to the + * savepoint, or `null`. A Postgres transaction accepts no statement after an error until then. When + * the database has already rolled back the whole transaction, e.g. a MySQL deadlock victim, the + * rollback to the savepoint fails and that exception propagates, so nothing is written outside + * the claim's transaction. Without a transaction there is nothing to protect, and no savepoint. + */ + private fun Connection.isolated(block: () -> Unit): SQLException? { + val savepoint = if (autoCommit) null else setSavepoint() + return try { + block() + if (savepoint != null) releaseSavepoint(savepoint) + null + } catch (e: SQLException) { + if (savepoint != null) { + try { + rollback(savepoint) + } catch (rollbackFailure: SQLException) { + rollbackFailure.addSuppressed(e) + throw rollbackFailure + } + } + e + } + } + private fun ResultSet.toOutboxEntry(): OutboxEntry = OutboxEntry( outboxId = OutboxId(UUID.fromString(getString("id"))), messageType = getString("message_type"), @@ -209,6 +266,12 @@ class PostgresOutboxStore( private fun ResultSet.getInstant(column: String): Instant? = getObject(column, OffsetDateTime::class.java)?.toInstant() companion object { + private val logger = LoggerFactory.getLogger(PostgresOutboxStore::class.java) + + /** Stored instead of a `last_error` the database rejected. ASCII, so that any charset accepts it. */ + private const val REJECTED_LAST_ERROR = "okapi: the database rejected this entry's error message; see the application log" + private const val LOGGED_ERROR_LENGTH = 2_000 + private val insertSql = """ INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, last_error, delivery_metadata, next_attempt_at) VALUES (?::uuid, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?) From f83a7bb5f87e83e6d1f02b0096198d4cb24b0369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Wokacz?= Date: Thu, 12 Nov 2026 17:02:48 +0100 Subject: [PATCH 2/2] fix(stores): review follow-up for rejected-row isolation (#141) - Use the placeholder only for a rejected value (SQLState class 22, MySQL 1366); rethrow deadlocks, timeouts and other failures. - When rolling back to the savepoint fails (e.g. a MySQL deadlock victim), rethrow the original exception with the rollback failure suppressed. - Release a savepoint after rolling back to it, so row savepoints don't nest. - Fall back to the plain batch when the connection refuses savepoints. - Placeholder carries the MySQL error code; WARN without a stack trace per row. - 005: explicit ALGORITHM=COPY, LOCK=SHARED; document the metadata-lock wait. - README: 005 in the migration list and notes; CHANGELOG: value errors only, Postgres subtransaction note. - Tests: 6-row batch (MySQL multi-statement rewrite), transient 40001 failure, real MySQL deadlock, charset tests on independent databases. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 ++-- README.md | 4 +- .../test/store/MysqlLastErrorCharsetTest.kt | 21 ++--- .../store/MysqlRejectedRowIsolationTest.kt | 63 +++++++++++++++ .../store/PostgresRejectedRowIsolationTest.kt | 3 + .../test/store/RejectedRowIsolationTests.kt | 32 ++++++-- .../okapi/mysql/MysqlOutboxStore.kt | 81 ++++++++++++------- .../db/mysql/005__last_error_utf8mb4.sql | 13 ++- .../okapi/postgres/PostgresOutboxStore.kt | 76 +++++++++++------ 9 files changed, 228 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b850290..fa88540c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,15 +38,20 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * When the batch update fails, `PostgresOutboxStore` and `MysqlOutboxStore` now roll back to a savepoint and write the rows one by one, each under its own savepoint. A row the database still rejects is written with a placeholder `last_error`, keeping its status and `retries`, and the original message is - logged at WARN with the entry id. A row rejected for a reason other than its error message still fails - the batch, as before. A successful batch costs two extra statements, `SAVEPOINT` and `RELEASE SAVEPOINT`. + logged at WARN with the entry id. Only a value error (SQLState class 22, or MySQL error 1366) gets the + placeholder; any other failure, such as a deadlock, a timeout or a lost connection, still fails the + batch as before. A successful batch costs two extra statements, `SAVEPOINT` and `RELEASE SAVEPOINT`. + On PostgreSQL the row-by-row fallback opens a subtransaction per row; above 64 rows that overflows the + per-backend subtransaction cache, so keep `batch-size` at or below 64 if rejected values are frequent. ([#141](https://github.com/softwaremill/okapi/issues/141)) - **MySQL: `last_error` is `utf8mb4` (new migration `005__last_error_utf8mb4`).** 001 created `okapi_outbox` in the schema's default character set. On a `latin1` or `utf8mb3` schema, common on servers upgraded from MySQL 5.7, an error message with an emoji or any other character outside that - set was rejected with `Incorrect string value`. 005 converts the column to `utf8mb4`. That rebuilds - the table, and writes, `publish()` included, wait until it finishes, so on a large outbox run it in a - quiet period. Its precondition skips it where the column is already `utf8mb4`, the MySQL 8 default. + set was rejected with `Incorrect string value`. 005 converts the column to `utf8mb4`. Its + precondition skips it where the column is already `utf8mb4`, the MySQL 8 default. Otherwise it copies + the table: it first waits up to 30 s for open transactions on `okapi_outbox`, queuing every other + statement on the table, and then blocks writes, `publish()` included, until the copy finishes. On a + large outbox, run it in a quiet period. ([#141](https://github.com/softwaremill/okapi/issues/141)) - **MySQL: the processor no longer blocks `publish()` for the duration of delivery.** Under MySQL's default `REPEATABLE READ`, the claim's `FOR UPDATE SKIP LOCKED` took gap locks on the claim index; diff --git a/README.md b/README.md index 05a382a0..87890bc9 100644 --- a/README.md +++ b/README.md @@ -223,12 +223,14 @@ If you use another migration tool, apply every SQL migration for your database, - **PostgreSQL:** [001](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql) route-aware claim index, [003](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql) `TIMESTAMPTZ` timestamp columns, [004](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/004__retry_backoff.sql) retry backoff column and claim index (its second changeset must run outside a transaction) - **MySQL:** [001](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql) route-aware claim index, [003](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql) `DATETIME(6)` timestamp columns, [004](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/004__retry_backoff.sql) retry backoff column and claim index, [005](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql) `utf8mb4` `last_error` (a no-op where it already is) -For an existing installation, apply every new migration (up to `004`) before starting updated workers. MySQL claims explicitly select the claim index from `004` and fail if it is missing. PostgreSQL's `002` and the index changeset of `004` use `CREATE INDEX CONCURRENTLY`, so run them outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. `CREATE INDEX CONCURRENTLY` waits for every transaction in the database that started before it, not only those on `okapi_outbox`, so long-running transactions (including old workers delivering a claimed batch) delay it, and with Liquibase they delay application startup while the changelog lock is held. Run it when no long transactions are open, or apply it ahead of the rollout. +For an existing installation, apply every new migration (up to `004`, and on MySQL `005`) before starting updated workers. MySQL claims explicitly select the claim index from `004` and fail if it is missing. PostgreSQL's `002` and the index changeset of `004` use `CREATE INDEX CONCURRENTLY`, so run them outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. `CREATE INDEX CONCURRENTLY` waits for every transaction in the database that started before it, not only those on `okapi_outbox`, so long-running transactions (including old workers delivering a claimed batch) delay it, and with Liquibase they delay application startup while the changelog lock is held. Run it when no long transactions are open, or apply it ahead of the rollout. PostgreSQL's `003` converts the timestamp columns from `TIMESTAMP` (the okapi JVM's local wall-clock time) to `TIMESTAMPTZ`. It reads existing values in the session `TimeZone`; pgjdbc sets that to the JVM default zone, so the bundled Liquibase changelog converts them correctly when it runs inside the application. When applying it by hand, run it inside a transaction (`psql -1`) after `SET TIME ZONE ''` in the same session. It takes an `ACCESS EXCLUSIVE` lock: it first waits (up to 30 s, then fails and retries on the next start) for open transactions on `okapi_outbox`, queuing every other statement on the table meanwhile, and then, unless the session zone is UTC (PostgreSQL 12+), rewrites the table while reads and writes wait. 1.x instances keep working after it, apart from one failed poll per pooled connection. See UPGRADING.md for when the in-app conversion uses the wrong zone. See [UPGRADING.md](UPGRADING.md#from-1x-to-200). MySQL's `003` converts the timestamp columns from `TIMESTAMP(6)`, which ends in January 2038, to `DATETIME(6)` holding UTC. Stop every 1.x instance, publishers included, before it runs: 1.x writes these columns in the JVM zone. It copies the table; it first waits (up to 30 s, then fails and retries on the next start) for open transactions on `okapi_outbox` to finish, queuing every other statement on the table meanwhile, and then blocks writes, including `publish()`, until the copy finishes. See [UPGRADING.md](UPGRADING.md#from-1x-to-200), including when rows written by 1.x can read back shifted. +MySQL's `005` converts `last_error` to `utf8mb4`, so that error messages with characters outside a `latin1` or `utf8mb3` schema's character set can be stored. Where the column is already `utf8mb4`, the MySQL 8 default, it does nothing. Otherwise it copies the table like `003`: it waits up to 30 s for open transactions on `okapi_outbox`, queuing every other statement on the table meanwhile, and then blocks writes, including `publish()`, until the copy finishes. Without it, okapi still works and stores a placeholder for an error message the column rejects. + okapi stores messages in the fixed `okapi_outbox` table. When the built-in Liquibase integration is used, it also uses two dedicated migration tracking tables: | Table | Purpose | diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorCharsetTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorCharsetTest.kt index 81c63b04..797abf8c 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorCharsetTest.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlLastErrorCharsetTest.kt @@ -58,7 +58,8 @@ class MysqlLastErrorCharsetTest : FunSpec({ container.start() DriverManager.getConnection(container.jdbcUrl, "root", container.password).use { conn -> conn.createStatement().use { - it.execute("CREATE DATABASE legacy CHARACTER SET latin1") + it.execute("CREATE DATABASE legacy_before_005 CHARACTER SET latin1") + it.execute("CREATE DATABASE legacy_upgraded CHARACTER SET latin1") it.execute("CREATE DATABASE modern") } } @@ -125,11 +126,11 @@ class MysqlLastErrorCharsetTest : FunSpec({ } test("before 005, a latin1 last_error rejects an emoji, and the store keeps the batch with a placeholder error") { - migrate("legacy", CHANGESETS_BEFORE_005) - lastErrorCharset("legacy") shouldBe "latin1" + migrate("legacy_before_005", CHANGESETS_BEFORE_005) + lastErrorCharset("legacy_before_005") shouldBe "latin1" // Control: the raw value really is rejected. shouldThrowAny { - connect("legacy").use { conn -> + connect("legacy_before_005").use { conn -> conn.prepareStatement( "INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, delivery_metadata, last_error) " + "VALUES (UUID(), 'm', '{}', 'charset-test', '{}', ?)", @@ -140,7 +141,7 @@ class MysqlLastErrorCharsetTest : FunSpec({ } } - val rows = processHealthyAndPoisoned("legacy") + val rows = processHealthyAndPoisoned("legacy_before_005") rows.getValue("healthy").first shouldBe OutboxStatus.DELIVERED rows.getValue("poisoned").first shouldBe OutboxStatus.PENDING @@ -148,11 +149,13 @@ class MysqlLastErrorCharsetTest : FunSpec({ } test("005 converts last_error to utf8mb4, which stores the emoji") { - migrate("legacy") - execType005("legacy") shouldBe "EXECUTED" - lastErrorCharset("legacy") shouldBe "utf8mb4" + migrate("legacy_upgraded", CHANGESETS_BEFORE_005) + lastErrorCharset("legacy_upgraded") shouldBe "latin1" + migrate("legacy_upgraded") + execType005("legacy_upgraded") shouldBe "EXECUTED" + lastErrorCharset("legacy_upgraded") shouldBe "utf8mb4" - val rows = processHealthyAndPoisoned("legacy") + val rows = processHealthyAndPoisoned("legacy_upgraded") rows.getValue("healthy").first shouldBe OutboxStatus.DELIVERED rows.getValue("poisoned") shouldBe (OutboxStatus.PENDING to EMOJI_ERROR) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlRejectedRowIsolationTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlRejectedRowIsolationTest.kt index 0851d0fd..389bff8f 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlRejectedRowIsolationTest.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlRejectedRowIsolationTest.kt @@ -1,9 +1,20 @@ package com.softwaremill.okapi.test.store +import com.softwaremill.okapi.core.DeliveryInfo +import com.softwaremill.okapi.core.OutboxEntry +import com.softwaremill.okapi.core.OutboxMessage +import com.softwaremill.okapi.core.OutboxStatus import com.softwaremill.okapi.mysql.MysqlOutboxStore import com.softwaremill.okapi.test.support.MysqlTestSupport +import io.kotest.assertions.throwables.shouldThrow import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import java.sql.Connection import java.sql.DriverManager +import java.sql.SQLException +import java.time.Instant +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit class MysqlRejectedRowIsolationTest : FunSpec({ val db = MysqlTestSupport() @@ -26,6 +37,9 @@ class MysqlRejectedRowIsolationTest : FunSpec({ IF NEW.last_error LIKE '%$REJECTED_ERROR_MARKER%' OR NEW.message_type = '$ALWAYS_REJECTED_TYPE' THEN SIGNAL SQLSTATE '22000' SET MESSAGE_TEXT = 'rejected by test trigger'; END IF; + IF NEW.message_type = '$TRANSIENT_FAILURE_TYPE' AND COALESCE(NEW.last_error, '') NOT LIKE 'okapi:%' THEN + SIGNAL SQLSTATE '40001' SET MESSAGE_TEXT = 'could not serialize (test trigger)'; + END IF; END """.trimIndent(), ) @@ -33,4 +47,53 @@ class MysqlRejectedRowIsolationTest : FunSpec({ } }, ) + + // A deadlock victim's whole transaction is rolled back, savepoints included: the store must not go + // on writing rows outside the claim's transaction, whose locks are gone. + test("[mysql] a deadlock during the batch update propagates, and no row is written") { + val store = MysqlOutboxStore(db.jdbc) + val now = Instant.now() + val (a, b) = listOf("a", "b").map { OutboxEntry.createPending(OutboxMessage(it, "{}"), DeadlockTestDeliveryInfo, now) } + val heavy = (1..50).map { OutboxEntry.createPending(OutboxMessage("heavy-$it", "{}"), DeadlockTestDeliveryInfo, now) } + db.jdbc.withTransaction { (listOf(a, b) + heavy).forEach { store.persist(it) } } + + DriverManager.getConnection(db.container.jdbcUrl, db.container.username, db.container.password).use { other -> + other.autoCommit = false + other.transactionIsolation = Connection.TRANSACTION_READ_COMMITTED + // Having changed more rows, the other transaction is never InnoDB's choice of deadlock victim. + other.createStatement().use { + it.executeUpdate( + "UPDATE okapi_outbox SET retries = retries + 1 WHERE message_type LIKE 'heavy-%'", + ) + } + other.lockRow(a) + + val failure = shouldThrow { + db.jdbc.withTransaction { + db.jdbc.withConnection { it.lockRow(b) } + // Whichever of the two waits first, they end up waiting for each other. + val otherWaitsForB = CompletableFuture.runAsync { runCatching { other.lockRow(b) } } + try { + store.updateAfterProcessingBatch(listOf(a, b).map { it.copy(status = OutboxStatus.DELIVERED) }) + } finally { + otherWaitsForB.get(30, TimeUnit.SECONDS) + } + } + } + other.rollback() + + failure.errorCode shouldBe 1213 + } + db.jdbc.withTransaction { store.countByStatuses() }[OutboxStatus.DELIVERED] shouldBe 0L + } }) + +private object DeadlockTestDeliveryInfo : DeliveryInfo { + override val type: String = "deadlock-test" + override fun serialize(): String = """{"type":"deadlock-test"}""" +} + +private fun Connection.lockRow(entry: OutboxEntry) = prepareStatement("SELECT id FROM okapi_outbox WHERE id = ? FOR UPDATE").use { + it.setString(1, entry.outboxId.raw.toString()) + it.executeQuery().close() +} diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresRejectedRowIsolationTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresRejectedRowIsolationTest.kt index 041c153d..7d80b11a 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresRejectedRowIsolationTest.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresRejectedRowIsolationTest.kt @@ -25,6 +25,9 @@ class PostgresRejectedRowIsolationTest : FunSpec({ IF NEW.last_error LIKE '%$REJECTED_ERROR_MARKER%' OR NEW.message_type = '$ALWAYS_REJECTED_TYPE' THEN RAISE EXCEPTION 'rejected by test trigger' USING ERRCODE = '22000'; END IF; + IF NEW.message_type = '$TRANSIENT_FAILURE_TYPE' AND COALESCE(NEW.last_error, '') NOT LIKE 'okapi:%' THEN + RAISE EXCEPTION 'could not serialize (test trigger)' USING ERRCODE = '40001'; + END IF; RETURN NEW; END ${'$'}${'$'} LANGUAGE plpgsql """.trimIndent(), diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/RejectedRowIsolationTests.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/RejectedRowIsolationTests.kt index a40e9612..1f912cdd 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/RejectedRowIsolationTests.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/RejectedRowIsolationTests.kt @@ -21,19 +21,23 @@ import java.time.Clock /** Marker in a `last_error` that the test trigger rejects. */ const val REJECTED_ERROR_MARKER = "REJECT_ME" -/** `message_type` whose rows the test trigger rejects on every update. */ +/** `message_type` whose rows the test trigger rejects on every update, as a rejected value (SQLState 22000). */ const val ALWAYS_REJECTED_TYPE = "always-rejected" +/** `message_type` whose rows the test trigger fails on every update with a serialization failure (SQLState 40001). */ +const val TRANSIENT_FAILURE_TYPE = "transient-failure" + private object IsolationTestDeliveryInfo : DeliveryInfo { override val type: String = "isolation-test" override fun serialize(): String = """{"type":"isolation-test"}""" } -/** Delivers `healthy` entries and fails all others with a retriable error carrying the trigger's marker. */ +/** Delivers `healthy` entries and fails the others; their error carries the trigger's marker, except a transient one's. */ private object HealthyOnlyDeliverer : MessageDeliverer { override val type: String = IsolationTestDeliveryInfo.type override fun deliver(entry: OutboxEntry): DeliveryResult = when { entry.messageType.startsWith("healthy") -> DeliveryResult.Success + entry.messageType == TRANSIENT_FAILURE_TYPE -> DeliveryResult.RetriableFailure("HTTP 503") else -> DeliveryResult.RetriableFailure("HTTP 503 $REJECTED_ERROR_MARKER") } } @@ -44,7 +48,9 @@ private data class IsolationRow(val status: OutboxStatus, val retries: Int, val * Issue #141: a row whose update the database rejects for any reason must not roll back the results * of the rest of its batch. [installRejectingTrigger] adds a BEFORE UPDATE trigger that rejects an * update whose `last_error` contains [REJECTED_ERROR_MARKER], or of a row whose `message_type` is - * [ALWAYS_REJECTED_TYPE], standing in for whatever the database might refuse. + * [ALWAYS_REJECTED_TYPE], standing in for whatever value the database might refuse, and fails an + * update of a [TRANSIENT_FAILURE_TYPE] row as a serialization failure would, unless it writes okapi's + * placeholder error, so that taking a transient failure for a rejected value would let the batch commit. */ fun FunSpec.rejectedRowIsolationTests( dbName: String, @@ -88,14 +94,14 @@ fun FunSpec.rejectedRowIsolationTests( ) test("[$dbName] a row whose last_error is rejected keeps its status and retries, and the rest of the batch commits") { - seed("healthy") + // More than 3 rows, so that Connector/J rewrites the MySQL batch into one multi-statement. + val healthy = (1..5).map { "healthy-$it" } + healthy.forEach { seed(it) } seed("poisoned") - seed("healthy-2") - jdbcProvider().withTransaction { processor().processNext(10) } shouldBe 3 + jdbcProvider().withTransaction { processor().processNext(10) } shouldBe 6 - row("healthy").status shouldBe OutboxStatus.DELIVERED - row("healthy-2").status shouldBe OutboxStatus.DELIVERED + healthy.forEach { row(it).status shouldBe OutboxStatus.DELIVERED } val poisoned = row("poisoned") poisoned.status shouldBe OutboxStatus.PENDING poisoned.retries shouldBe 1 @@ -112,6 +118,16 @@ fun FunSpec.rejectedRowIsolationTests( row(ALWAYS_REJECTED_TYPE).retries shouldBe 0 } + test("[$dbName] a transient failure is not taken for a rejected value: the batch fails, and nothing is written") { + seed("healthy") + seed(TRANSIENT_FAILURE_TYPE) + + shouldThrow { jdbcProvider().withTransaction { processor().processNext(10) } }.sqlState shouldBe "40001" + + row("healthy").status shouldBe OutboxStatus.PENDING + row(TRANSIENT_FAILURE_TYPE) shouldBe IsolationRow(OutboxStatus.PENDING, 0, null) + } + test("[$dbName] a batch with no rejected row is written as before") { seed("healthy") seed("healthy-2") diff --git a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt index a32bca57..f65a7e63 100644 --- a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt +++ b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt @@ -182,59 +182,83 @@ class MysqlOutboxStore( } stmt.executeBatch() } - } - if (batchFailure != null) { - logger.debug("Batch update of {} outbox entries failed; writing them one by one", entries.size, batchFailure) + } ?: return@withConnection + logger.debug("Batch update of {} outbox entries failed, writing them one by one: {}", entries.size, batchFailure.toString()) + try { entries.forEach { conn.updateRowByRow(it) } + } catch (e: SQLException) { + e.addSuppressed(batchFailure) + throw e } } return entries } /** - * Writes [entry] on its own. If the database rejects it and it has a `last_error`, the most likely - * culprit, it is written again with a placeholder error, so that its status and retries are kept. + * Writes [entry] on its own. If the database rejects a value in it (an SQL data exception) and it + * has a `last_error`, the likely culprit, it is written again with a placeholder error, so that + * its status and retries are kept. Any other failure propagates. */ private fun Connection.updateRowByRow(entry: OutboxEntry) { val rejected = isolated { prepareStatement(updateSql).use { it.prepareForUpdateFrom(entry).executeUpdate() } } ?: return - val lastError = entry.lastError ?: throw rejected - logger.warn( - "The database rejected the last_error of outbox entry {}; storing a placeholder. Rejected error (truncated): {}", - entry.outboxId.raw, - lastError.take(LOGGED_ERROR_LENGTH), - rejected, - ) - val placeholder = entry.copy(lastError = "$REJECTED_LAST_ERROR (SQLState ${rejected.sqlState})") + val lastError = entry.lastError + if (lastError == null || !rejected.isRejectedValue()) throw rejected + val placeholder = entry.copy(lastError = "$REJECTED_LAST_ERROR (SQLState ${rejected.sqlState}, error code ${rejected.errorCode})") isolated { prepareStatement(updateSql).use { it.prepareForUpdateFrom(placeholder).executeUpdate() } }?.let { it.addSuppressed(rejected) throw it } + logger.warn( + "The database rejected the last_error of outbox entry {} ({}); stored a placeholder instead. " + + "First {} characters of the rejected error: {}", + entry.outboxId.raw, + rejected.toString(), + LOGGED_ERROR_LENGTH, + lastError.take(LOGGED_ERROR_LENGTH), + ) } + /** A value the database refuses to store, as opposed to a transient or transaction-level failure. */ + private fun SQLException.isRejectedValue(): Boolean = sqlState?.startsWith("22") == true || errorCode in REJECTED_VALUE_ERROR_CODES + /** * Runs [block] under a savepoint and returns the [SQLException] it threw, after rolling back to the * savepoint, or `null`. A Postgres transaction accepts no statement after an error until then. When * the database has already rolled back the whole transaction, e.g. a MySQL deadlock victim, the - * rollback to the savepoint fails and that exception propagates, so nothing is written outside - * the claim's transaction. Without a transaction there is nothing to protect, and no savepoint. + * rollback to the savepoint fails and the original exception propagates, so nothing is written + * outside the claim's transaction. Without a transaction there is nothing to protect, and no + * savepoint. A connection that refuses savepoints runs [block] as is, and its failure propagates. */ private fun Connection.isolated(block: () -> Unit): SQLException? { - val savepoint = if (autoCommit) null else setSavepoint() - return try { + if (autoCommit) { + return try { + block() + null + } catch (e: SQLException) { + e + } + } + val savepoint = try { + setSavepoint() + } catch (_: SQLException) { + block() + return null + } + try { block() - if (savepoint != null) releaseSavepoint(savepoint) - null } catch (e: SQLException) { - if (savepoint != null) { - try { - rollback(savepoint) - } catch (rollbackFailure: SQLException) { - rollbackFailure.addSuppressed(e) - throw rollbackFailure - } + try { + rollback(savepoint) + // Released, so that the next savepoint is not nested under this one. + releaseSavepoint(savepoint) + } catch (rollbackFailure: SQLException) { + e.addSuppressed(rollbackFailure) + throw e } - e + return e } + releaseSavepoint(savepoint) + return null } private fun ResultSet.toOutboxEntry(): OutboxEntry = OutboxEntry( @@ -282,6 +306,9 @@ class MysqlOutboxStore( private const val REJECTED_LAST_ERROR = "okapi: the database rejected this entry's error message; see the application log" private const val LOGGED_ERROR_LENGTH = 2_000 + /** Errors for a value a column cannot hold that MySQL reports with SQLState HY000: 1366, incorrect string value. */ + private val REJECTED_VALUE_ERROR_CODES = setOf(1366) + private val upsertSql = """ INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, last_error, delivery_metadata, next_attempt_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) diff --git a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql index 36f3e9c3..4d9576bd 100644 --- a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql +++ b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/005__last_error_utf8mb4.sql @@ -6,14 +6,19 @@ -- "Incorrect string value". The store then writes a placeholder instead, so the batch survives, but the -- message is lost. utf8mb4 stores every character. -- --- Changing a column's character set rebuilds the table, and writes, publish() included, wait until it --- finishes. The precondition skips the rebuild where last_error is already utf8mb4: the MySQL 8 default, --- so on most installations this does nothing. +-- The ALTER copies the table. It first waits for a metadata lock behind every open transaction that +-- touched okapi_outbox, such as a worker delivering a claimed batch, while every new statement on the +-- table, reads included, queues behind it. lock_wait_timeout bounds that wait to 30 s; on timeout this +-- migration fails and runs again on the next start. Writes, publish() included, then wait until the copy +-- finishes. The precondition skips all this where last_error is already utf8mb4, as on a schema created +-- by MySQL 8 with its default character set. Schemas created by MySQL 5.7, or by MariaDB builds that +-- default to latin1, are converted. --changeset outbox:005 --preconditions onFail:MARK_RAN --precondition-sql-check expectedResult:0 SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'okapi_outbox' AND COLUMN_NAME = 'last_error' AND CHARACTER_SET_NAME = 'utf8mb4' SET @okapi_previous_lock_wait_timeout = @@session.lock_wait_timeout; SET SESSION lock_wait_timeout = 30; -ALTER TABLE okapi_outbox MODIFY last_error TEXT CHARACTER SET utf8mb4 NULL; +ALTER TABLE okapi_outbox MODIFY last_error TEXT CHARACTER SET utf8mb4 NULL, + ALGORITHM=COPY, LOCK=SHARED; SET SESSION lock_wait_timeout = @okapi_previous_lock_wait_timeout; diff --git a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt index 8ba37e61..32f93b72 100644 --- a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt +++ b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt @@ -170,59 +170,83 @@ class PostgresOutboxStore( } stmt.executeBatch() } - } - if (batchFailure != null) { - logger.debug("Batch update of {} outbox entries failed; writing them one by one", entries.size, batchFailure) + } ?: return@withConnection + logger.debug("Batch update of {} outbox entries failed, writing them one by one: {}", entries.size, batchFailure.toString()) + try { entries.forEach { conn.updateRowByRow(it) } + } catch (e: SQLException) { + e.addSuppressed(batchFailure) + throw e } } return entries } /** - * Writes [entry] on its own. If the database rejects it and it has a `last_error`, the most likely - * culprit, it is written again with a placeholder error, so that its status and retries are kept. + * Writes [entry] on its own. If the database rejects a value in it (an SQL data exception) and it + * has a `last_error`, the likely culprit, it is written again with a placeholder error, so that + * its status and retries are kept. Any other failure propagates. */ private fun Connection.updateRowByRow(entry: OutboxEntry) { val rejected = isolated { prepareStatement(updateSql).use { it.prepareForUpdateFrom(entry).executeUpdate() } } ?: return - val lastError = entry.lastError ?: throw rejected - logger.warn( - "The database rejected the last_error of outbox entry {}; storing a placeholder. Rejected error (truncated): {}", - entry.outboxId.raw, - lastError.take(LOGGED_ERROR_LENGTH), - rejected, - ) + val lastError = entry.lastError + if (lastError == null || !rejected.isRejectedValue()) throw rejected val placeholder = entry.copy(lastError = "$REJECTED_LAST_ERROR (SQLState ${rejected.sqlState})") isolated { prepareStatement(updateSql).use { it.prepareForUpdateFrom(placeholder).executeUpdate() } }?.let { it.addSuppressed(rejected) throw it } + logger.warn( + "The database rejected the last_error of outbox entry {} ({}); stored a placeholder instead. " + + "First {} characters of the rejected error: {}", + entry.outboxId.raw, + rejected.toString(), + LOGGED_ERROR_LENGTH, + lastError.take(LOGGED_ERROR_LENGTH), + ) } + /** A value the database refuses to store, as opposed to a transient or transaction-level failure. */ + private fun SQLException.isRejectedValue(): Boolean = sqlState?.startsWith("22") == true + /** * Runs [block] under a savepoint and returns the [SQLException] it threw, after rolling back to the * savepoint, or `null`. A Postgres transaction accepts no statement after an error until then. When * the database has already rolled back the whole transaction, e.g. a MySQL deadlock victim, the - * rollback to the savepoint fails and that exception propagates, so nothing is written outside - * the claim's transaction. Without a transaction there is nothing to protect, and no savepoint. + * rollback to the savepoint fails and the original exception propagates, so nothing is written + * outside the claim's transaction. Without a transaction there is nothing to protect, and no + * savepoint. A connection that refuses savepoints runs [block] as is, and its failure propagates. */ private fun Connection.isolated(block: () -> Unit): SQLException? { - val savepoint = if (autoCommit) null else setSavepoint() - return try { + if (autoCommit) { + return try { + block() + null + } catch (e: SQLException) { + e + } + } + val savepoint = try { + setSavepoint() + } catch (_: SQLException) { + block() + return null + } + try { block() - if (savepoint != null) releaseSavepoint(savepoint) - null } catch (e: SQLException) { - if (savepoint != null) { - try { - rollback(savepoint) - } catch (rollbackFailure: SQLException) { - rollbackFailure.addSuppressed(e) - throw rollbackFailure - } + try { + rollback(savepoint) + // Released, so that the next savepoint is not nested under this one. + releaseSavepoint(savepoint) + } catch (rollbackFailure: SQLException) { + e.addSuppressed(rollbackFailure) + throw e } - e + return e } + releaseSavepoint(savepoint) + return null } private fun ResultSet.toOutboxEntry(): OutboxEntry = OutboxEntry(