From 443522f3a04e24d9456ae4db737e3a8994327384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Wokacz?= Date: Thu, 5 Nov 2026 16:23:11 +0100 Subject: [PATCH 1/2] fix(mysql)!: store timestamps as UTC DATETIME(6) so the outbox survives 2038 (#138) MySQL TIMESTAMP ends at 2038-01-19 03:14:07 UTC, after which every publish() fails. Changeset 003 converts created_at, updated_at and last_attempt to DATETIME(6) holding UTC, pinning the session time zone to UTC for the conversion and restoring it afterwards. MysqlOutboxStore binds and reads the columns as UTC LocalDateTime, which Connector/J passes through unconverted, so stored values no longer depend on the JVM, session or driver zone. BREAKING CHANGE: new migration 003 rebuilds okapi_outbox on MySQL and must be applied before starting updated workers; the columns now hold UTC wall-clock DATETIME values. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 ++ README.md | 10 +- UPGRADING.md | 15 +++ .../test/store/MysqlDatetimeMigrationTest.kt | 90 ++++++++++++++++ .../test/store/MysqlTimestampColumnsTest.kt | 20 ++++ .../okapi/test/store/TimestampColumnTests.kt | 102 ++++++++++++++++++ .../okapi/mysql/MysqlOutboxStore.kt | 43 ++++---- .../okapi/db/mysql/003__datetime_columns.sql | 18 ++++ .../softwaremill/okapi/db/mysql/changelog.xml | 1 + 9 files changed, 282 insertions(+), 27 deletions(-) create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlDatetimeMigrationTest.kt create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlTimestampColumnsTest.kt create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/TimestampColumnTests.kt create mode 100644 okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 94178b2b..3be97363 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,16 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * yourself, apply it before starting updated workers: the MySQL claim uses `FORCE INDEX` and fails without it, and the PostgreSQL script must run outside a transaction (`CREATE INDEX CONCURRENTLY`). ([#120](https://github.com/softwaremill/okapi/pull/120)) +- **MySQL: timestamp columns are `DATETIME(6)` in UTC (new migration `003__datetime_columns`).** + `created_at`, `updated_at` and `last_attempt` were `TIMESTAMP(6)`, which cannot hold anything + after `2038-01-19 03:14:07 UTC`; from then on every `publish()` would fail. The migration converts + existing values without shifting them (it pins the session to UTC for the conversion) and drops the + `CURRENT_TIMESTAMP` column defaults, which okapi never relied on. `MysqlOutboxStore` now binds and + reads these columns as UTC `LocalDateTime`, so stored values no longer depend on the JVM time zone, + the session `time_zone` or Connector/J's `connectionTimeZone`/`preserveInstants`. The migration + rebuilds the table and blocks writes to it while it runs; apply it before starting updated workers. + Anything else reading these columns directly must now treat them as UTC wall-clock values. + ([#138](https://github.com/softwaremill/okapi/issues/138)) ## [1.0.0] — 2026-07-28 diff --git a/README.md b/README.md index 7be4fa3f..8081935d 100644 --- a/README.md +++ b/README.md @@ -213,15 +213,15 @@ okapi ships Liquibase changelogs that create its table and indexes: With `okapi-spring-boot` and Liquibase on the classpath, these run automatically against the configured `DataSource` at startup, tracked in dedicated Liquibase tables by default to avoid conflicts with the application's migration history. -If you use another migration tool, apply both SQL migrations for your database in order: +If you use another migration tool, apply every SQL migration for your database, in order: -| Database | 001: table and initial indexes | 002: route-aware claim index | -|---|---|---| -| PostgreSQL | [001 SQL](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/001__create_okapi_outbox_table.sql) | [002 SQL](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql) | -| MySQL | [001 SQL](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/001__create_okapi_outbox_table.sql) | [002 SQL](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql) | +- **PostgreSQL:** [001](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql) route-aware claim index +- **MySQL:** [001](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql) route-aware claim index, [003](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql) `DATETIME(6)` timestamp columns For an existing installation, apply `002` before starting updated workers. MySQL claims explicitly select the new index and fail if it is missing. PostgreSQL's `002` uses `CREATE INDEX CONCURRENTLY`, so run it outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. `CREATE INDEX CONCURRENTLY` waits for every transaction in the database that started before it, not only those on `okapi_outbox`, so long-running transactions (including old workers delivering a claimed batch) delay it, and with Liquibase they delay application startup while the changelog lock is held. Run it when no long transactions are open, or apply it ahead of the rollout. +MySQL's `003` converts the timestamp columns from `TIMESTAMP(6)`, which ends in January 2038, to `DATETIME(6)` holding UTC. It rebuilds the table, so writes to `okapi_outbox`, including `publish()`, wait until it finishes; on a large table, purge delivered rows first or apply it in a maintenance window. Apply it before starting updated workers: they read and write the columns as UTC `DATETIME` values. + okapi stores messages in the fixed `okapi_outbox` table. When the built-in Liquibase integration is used, it also uses two dedicated migration tracking tables: | Table | Purpose | diff --git a/UPGRADING.md b/UPGRADING.md index b92aabd7..9751c655 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -3,6 +3,21 @@ This guide contains actions required when upgrading between incompatible okapi versions. For the full release history, see [CHANGELOG.md](CHANGELOG.md). +## From 1.x to 2.0.0 + +### MySQL: `DATETIME(6)` timestamp columns (#138) + +Changeset `003__datetime_columns` converts `created_at`, `updated_at` and `last_attempt` from +`TIMESTAMP(6)` to `DATETIME(6)` holding UTC. With the bundled Liquibase changelog it runs +automatically at startup; otherwise apply +[`003__datetime_columns.sql`](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql) +yourself, **including its `SET time_zone` statements**, before starting 2.0.0 workers. + +- The `ALTER TABLE` rebuilds `okapi_outbox` and blocks writes to it, including `publish()`, until + it finishes. Purge delivered rows first or schedule it for a quiet period on a large table. +- Queries or dashboards that read these columns directly now get UTC wall-clock values, not values + converted to the session time zone. + ## From 0.2.x to 1.0.0 Releases up to 0.2.x used the domain table `outbox` and the shared Liquibase tracking diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlDatetimeMigrationTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlDatetimeMigrationTest.kt new file mode 100644 index 00000000..6cf29ee7 --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlDatetimeMigrationTest.kt @@ -0,0 +1,90 @@ +package com.softwaremill.okapi.test.store + +import com.mysql.cj.jdbc.MysqlDataSource +import com.softwaremill.okapi.core.OutboxStatus +import com.softwaremill.okapi.mysql.MysqlOutboxStore +import com.softwaremill.okapi.test.support.JdbcConnectionProvider +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import liquibase.Liquibase +import liquibase.database.DatabaseFactory +import liquibase.database.jvm.JdbcConnection +import liquibase.resource.ClassLoaderResourceAccessor +import org.testcontainers.containers.MySQLContainer +import java.sql.Connection +import java.sql.DriverManager +import java.time.Instant +import java.util.Properties + +private const val CHANGELOG = "com/softwaremill/okapi/db/mysql/changelog.xml" + +/** + * Upgrade path for issue #138: rows written while the columns were TIMESTAMP must keep their + * instants when changeset 003 converts them to DATETIME, even though the migrating session runs in + * another time zone (MODIFY converts through the session zone). + */ +class MysqlDatetimeMigrationTest : FunSpec({ + val container = MySQLContainer("mysql:8.0") + + beforeSpec { container.start() } + afterSpec { container.stop() } + + /** A connection whose session time zone is +05:00, as a non-UTC server or pool would give. */ + fun offsetSessionConnection(): Connection = DriverManager.getConnection( + container.jdbcUrl, + Properties().apply { + setProperty("user", container.username) + setProperty("password", container.password) + setProperty("sessionVariables", "time_zone='+05:00'") + }, + ) + + test("changeset 003 keeps existing instants and restores the session time zone") { + offsetSessionConnection().use { conn -> + val liquibase = + Liquibase( + CHANGELOG, + ClassLoaderResourceAccessor(), + DatabaseFactory.getInstance().findCorrectDatabaseImplementation(JdbcConnection(conn)), + ) + liquibase.update(2, "") + + // A TIMESTAMP literal is read in the session zone: 17:00 at +05:00 is 12:00Z. + conn.createStatement().use { + it.execute( + """ + INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, delivery_metadata) + VALUES ('00000000-0000-0000-0000-000000000001', 'm', '{}', 'http', 'PENDING', + '2024-06-01 17:00:00.250000', '2024-06-01 17:00:01', 1, '2024-06-01 17:00:02', '{}') + """.trimIndent(), + ) + } + + liquibase.update("") + + conn.createStatement().use { stmt -> + stmt.executeQuery( + "SELECT @@session.time_zone AS tz, DATA_TYPE AS t FROM information_schema.COLUMNS " + + "WHERE TABLE_NAME = 'okapi_outbox' AND COLUMN_NAME = 'created_at'", + ).use { rs -> + rs.next() + rs.getString("tz") shouldBe "+05:00" + rs.getString("t") shouldBe "datetime" + } + } + } + + val jdbc = JdbcConnectionProvider( + MysqlDataSource().apply { + setURL(container.jdbcUrl) + user = container.username + setPassword(container.password) + }, + ) + val entry = jdbc.withTransaction { MysqlOutboxStore(jdbc).claimPending(10) }.single() + entry.createdAt shouldBe Instant.parse("2024-06-01T12:00:00.25Z") + entry.updatedAt shouldBe Instant.parse("2024-06-01T12:00:01Z") + entry.lastAttempt shouldBe Instant.parse("2024-06-01T12:00:02Z") + entry.status shouldBe OutboxStatus.PENDING + } +}) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlTimestampColumnsTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlTimestampColumnsTest.kt new file mode 100644 index 00000000..caa483b8 --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlTimestampColumnsTest.kt @@ -0,0 +1,20 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.mysql.MysqlOutboxStore +import com.softwaremill.okapi.test.support.MysqlTestSupport +import io.kotest.core.spec.style.FunSpec + +class MysqlTimestampColumnsTest : FunSpec({ + val db = MysqlTestSupport() + + timestampColumnTests( + dbName = "mysql", + jdbcProvider = { db.jdbc }, + storeFactory = { MysqlOutboxStore(db.jdbc) }, + startDb = { db.start() }, + stopDb = { db.stop() }, + truncate = { db.truncate() }, + utcText = { column -> "DATE_FORMAT($column, '%Y-%m-%d %H:%i:%s.%f')" }, + setSessionZone = "SET time_zone = '+05:00'", + ) +}) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/TimestampColumnTests.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/TimestampColumnTests.kt new file mode 100644 index 00000000..9c1fe65b --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/TimestampColumnTests.kt @@ -0,0 +1,102 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.core.DeliveryInfo +import com.softwaremill.okapi.core.OutboxEntry +import com.softwaremill.okapi.core.OutboxMessage +import com.softwaremill.okapi.core.OutboxStatus +import com.softwaremill.okapi.core.OutboxStore +import com.softwaremill.okapi.test.support.JdbcConnectionProvider +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.collections.shouldContainExactly +import io.kotest.matchers.shouldBe +import java.time.Instant +import java.util.TimeZone + +private object TimestampTestDeliveryInfo : DeliveryInfo { + override val type: String = "timestamp-test" + override fun serialize(): String = """{"type":"timestamp-test"}""" +} + +/** + * Timestamp columns must hold instants that survive any JVM, session or driver time zone, and dates + * past 2038 (issues #135, #138). [utcText] renders a column as UTC `yyyy-MM-dd HH:mm:ss.SSSSSS` in + * SQL, so the tests check what is stored, not only what the store reads back. [setSessionZone] is a + * statement that moves the session time zone away from UTC. + */ +fun FunSpec.timestampColumnTests( + dbName: String, + jdbcProvider: () -> JdbcConnectionProvider, + storeFactory: () -> OutboxStore, + startDb: () -> Unit, + stopDb: () -> Unit, + truncate: () -> Unit, + utcText: (column: String) -> String, + setSessionZone: String, +) { + beforeSpec { startDb() } + afterSpec { stopDb() } + beforeEach { truncate() } + + fun storedUtc(id: String, column: String): String = jdbcProvider().withTransaction { + jdbcProvider().withConnection { conn -> + conn.prepareStatement("SELECT ${utcText(column)} AS v FROM okapi_outbox WHERE CAST(id AS CHAR(36)) = ?").use { stmt -> + stmt.setString(1, id) + stmt.executeQuery().use { rs -> + check(rs.next()) { "no row $id" } + rs.getString("v") + } + } + } + } + + fun entry(createdAt: Instant): OutboxEntry = + OutboxEntry.createPending(OutboxMessage("timestamp.test", "{}"), TimestampTestDeliveryInfo, createdAt) + + test("[$dbName] instants after 2038 are stored and read back") { + val store = storeFactory() + val createdAt = Instant.parse("2040-02-29T12:34:56.123456Z") + val attempted = entry(createdAt).retry(Instant.parse("2100-01-01T00:00:00.000001Z"), "boom") + jdbcProvider().withTransaction { store.persist(attempted) } + + val claimed = jdbcProvider().withTransaction { store.claimPending(10) } + claimed.map { it.createdAt } shouldContainExactly listOf(createdAt) + claimed.single().lastAttempt shouldBe Instant.parse("2100-01-01T00:00:00.000001Z") + storedUtc(attempted.outboxId.raw.toString(), "created_at") shouldBe "2040-02-29 12:34:56.123456" + } + + test("[$dbName] the purger cutoff works past 2038") { + val store = storeFactory() + val delivered = entry(Instant.parse("2045-01-01T00:00:00Z")).toDelivered(Instant.parse("2045-01-01T00:00:01Z")) + jdbcProvider().withTransaction { store.persist(delivered) } + + jdbcProvider().withTransaction { store.removeDeliveredBefore(Instant.parse("2045-01-01T00:00:00Z"), 10) } shouldBe 0 + jdbcProvider().withTransaction { store.removeDeliveredBefore(Instant.parse("2045-01-01T00:00:02Z"), 10) } shouldBe 1 + } + + test("[$dbName] values are stored as UTC whatever the JVM and session time zones") { + val store = storeFactory() + // Europe/Warsaw falls back at 01:00Z on this day: 00:30Z and 01:30Z are both 02:30 local, so a + // value stored as local wall-clock time reads back an hour early. + val createdAt = Instant.parse("2024-10-27T01:30:00.5Z") + val original = TimeZone.getDefault() + val pending = entry(createdAt) + try { + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Warsaw")) + jdbcProvider().withTransaction { + jdbcProvider().withConnection { conn -> conn.createStatement().use { it.execute(setSessionZone) } } + store.persist(pending) + } + storedUtc(pending.outboxId.raw.toString(), "created_at") shouldBe "2024-10-27 01:30:00.500000" + + TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) + val readBack = jdbcProvider().withTransaction { + jdbcProvider().withConnection { conn -> conn.createStatement().use { it.execute(setSessionZone) } } + store.findOldestCreatedAt(setOf(OutboxStatus.PENDING)) + } + readBack shouldBe mapOf(OutboxStatus.PENDING to createdAt) + jdbcProvider().withTransaction { store.claimPending(10) }.single().createdAt shouldBe createdAt + } finally { + TimeZone.setDefault(original) + } + } +} diff --git a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt index b7713a52..1f8802ef 100644 --- a/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt +++ b/okapi-mysql/src/main/kotlin/com/softwaremill/okapi/mysql/MysqlOutboxStore.kt @@ -7,9 +7,10 @@ import com.softwaremill.okapi.core.OutboxStatus import com.softwaremill.okapi.core.RouteAwareOutboxStore import java.sql.PreparedStatement import java.sql.ResultSet -import java.sql.Timestamp import java.sql.Types import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneOffset import java.util.UUID /** MySQL [OutboxStore] implementation using plain JDBC. */ @@ -25,17 +26,10 @@ class MysqlOutboxStore( stmt.setString(3, entry.payload) stmt.setString(4, entry.deliveryType) stmt.setString(5, entry.status.name) - stmt.setTimestamp(6, Timestamp.from(entry.createdAt)) - stmt.setTimestamp(7, Timestamp.from(entry.updatedAt)) + stmt.setUtc(6, entry.createdAt) + stmt.setUtc(7, entry.updatedAt) stmt.setInt(8, entry.retries) - if (entry.lastAttempt != null) { - stmt.setTimestamp( - 9, - Timestamp.from(entry.lastAttempt), - ) - } else { - stmt.setNull(9, Types.TIMESTAMP) - } + stmt.setUtc(9, entry.lastAttempt) val lastError = entry.lastError if (lastError != null) stmt.setString(10, OutboxEntry.sanitizeLastError(lastError)) else stmt.setNull(10, Types.VARCHAR) stmt.setString(11, entry.deliveryMetadata) @@ -126,7 +120,7 @@ class MysqlOutboxStore( return connectionProvider.withConnection { conn -> conn.prepareStatement(sql).use { stmt -> stmt.setString(1, OutboxStatus.DELIVERED.name) - stmt.setTimestamp(2, Timestamp.from(time)) + stmt.setUtc(2, time) stmt.setInt(3, limit) stmt.executeUpdate() } @@ -146,7 +140,7 @@ class MysqlOutboxStore( stmt.executeQuery().use { rs -> while (rs.next()) { val s = OutboxStatus.from(rs.getString("status")) - result[s] = rs.getTimestamp("min_created_at").toInstant() + result[s] = checkNotNull(rs.getUtc("min_created_at")) } } } @@ -189,23 +183,19 @@ class MysqlOutboxStore( payload = getString("payload"), deliveryType = getString("delivery_type"), status = OutboxStatus.from(getString("status")), - createdAt = getTimestamp("created_at").toInstant(), - updatedAt = getTimestamp("updated_at").toInstant(), + createdAt = checkNotNull(getUtc("created_at")), + updatedAt = checkNotNull(getUtc("updated_at")), retries = getInt("retries"), - lastAttempt = getTimestamp("last_attempt")?.toInstant(), + lastAttempt = getUtc("last_attempt"), lastError = getString("last_error"), deliveryMetadata = getString("delivery_metadata"), ) private fun PreparedStatement.prepareForUpdateFrom(entry: OutboxEntry): PreparedStatement { setString(1, entry.status.name) - setTimestamp(2, Timestamp.from(entry.updatedAt)) + setUtc(2, entry.updatedAt) setInt(3, entry.retries) - if (entry.lastAttempt != null) { - setTimestamp(4, Timestamp.from(entry.lastAttempt)) - } else { - setNull(4, Types.TIMESTAMP) - } + setUtc(4, entry.lastAttempt) val lastError = entry.lastError if (lastError != null) setString(5, OutboxEntry.sanitizeLastError(lastError)) else setNull(5, Types.VARCHAR) // CHAR(36) column — MySQL has no native UUID type, so the id travels as a string. @@ -213,6 +203,15 @@ class MysqlOutboxStore( return this } + // The DATETIME(6) columns hold UTC wall-clock time (issue #138). Connector/J passes LocalDateTime + // through without time-zone conversion, unlike java.sql.Timestamp, which it shifts by the JVM or + // connectionTimeZone zone -- so values stay UTC whatever the JVM, session or driver settings. + private fun PreparedStatement.setUtc(index: Int, instant: Instant?) { + if (instant != null) setObject(index, LocalDateTime.ofInstant(instant, ZoneOffset.UTC)) else setNull(index, Types.TIMESTAMP) + } + + private fun ResultSet.getUtc(column: String): Instant? = getObject(column, LocalDateTime::class.java)?.toInstant(ZoneOffset.UTC) + companion object { private val upsertSql = """ INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, last_error, delivery_metadata) diff --git a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql new file mode 100644 index 00000000..eaae7ba0 --- /dev/null +++ b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql @@ -0,0 +1,18 @@ +--liquibase formatted sql +--changeset outbox:003 + +-- TIMESTAMP stops at 2038-01-19 03:14:07 UTC (issue #138). DATETIME(6) has no such limit and, +-- unlike TIMESTAMP, is not converted through the session time zone: okapi stores UTC wall-clock +-- values in it. MODIFY converts the existing TIMESTAMP values as the session time zone renders +-- them, so pin the session to UTC for the conversion and restore it afterwards (Liquibase may run +-- on a pooled connection that the application reuses). This rebuilds the table, and writes to it +-- (including publish()) wait until the rebuild finishes. +SET @okapi_previous_time_zone = @@session.time_zone; +SET time_zone = '+00:00'; + +ALTER TABLE okapi_outbox + MODIFY created_at DATETIME(6) NOT NULL, + MODIFY updated_at DATETIME(6) NOT NULL, + MODIFY last_attempt DATETIME(6) NULL; + +SET time_zone = @okapi_previous_time_zone; diff --git a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml index 571916b6..12d53f91 100644 --- a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml +++ b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/changelog.xml @@ -5,4 +5,5 @@ http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd"> + From 65decfd1651ffd9a1966a8a10979606966dd2179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Wokacz?= Date: Thu, 5 Nov 2026 16:53:50 +0100 Subject: [PATCH 2/2] fix(mysql): address review of the DATETIME migration (#138) - Bound the metadata-lock wait of the ALTER with lock_wait_timeout = 30 and restore it, and make ALGORITHM=COPY, LOCK=SHARED explicit; the header comment now says settings are restored on success only. - Document that every 1.x instance must be stopped before 003 runs, that rows 1.x wrote while the JVM and MySQL session zones differed read back shifted (drain first), and the real locking behaviour, in UPGRADING, README and CHANGELOG. - The migration test checks type, precision, nullability, defaults and EXTRA of all three columns and that lock_wait_timeout is restored. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 +++-- README.md | 2 +- UPGRADING.md | 21 +++++++++- .../test/store/MysqlDatetimeMigrationTest.kt | 42 ++++++++++++++++--- .../okapi/db/mysql/003__datetime_columns.sql | 18 ++++++-- 5 files changed, 78 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3be97363..45c63f2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -144,9 +144,13 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * existing values without shifting them (it pins the session to UTC for the conversion) and drops the `CURRENT_TIMESTAMP` column defaults, which okapi never relied on. `MysqlOutboxStore` now binds and reads these columns as UTC `LocalDateTime`, so stored values no longer depend on the JVM time zone, - the session `time_zone` or Connector/J's `connectionTimeZone`/`preserveInstants`. The migration - rebuilds the table and blocks writes to it while it runs; apply it before starting updated workers. - Anything else reading these columns directly must now treat them as UTC wall-clock values. + the session `time_zone` or Connector/J's `connectionTimeZone`/`preserveInstants`. Stop every 1.x + instance, publishers included, before the migration runs (1.x writes these columns in the JVM + zone), and drain the outbox first if your JVMs and MySQL session ran in different zones: rows 1.x + wrote then were stored shifted by the difference. The migration copies the table, waits at most + 30 s for a metadata lock (then fails and retries on the next start) and blocks writes while it + copies. Anything else reading these columns directly must now treat them as UTC wall-clock values. + See UPGRADING.md. ([#138](https://github.com/softwaremill/okapi/issues/138)) ## [1.0.0] — 2026-07-28 diff --git a/README.md b/README.md index 8081935d..91307e62 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ If you use another migration tool, apply every SQL migration for your database, For an existing installation, apply `002` before starting updated workers. MySQL claims explicitly select the new index and fail if it is missing. PostgreSQL's `002` uses `CREATE INDEX CONCURRENTLY`, so run it outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. `CREATE INDEX CONCURRENTLY` waits for every transaction in the database that started before it, not only those on `okapi_outbox`, so long-running transactions (including old workers delivering a claimed batch) delay it, and with Liquibase they delay application startup while the changelog lock is held. Run it when no long transactions are open, or apply it ahead of the rollout. -MySQL's `003` converts the timestamp columns from `TIMESTAMP(6)`, which ends in January 2038, to `DATETIME(6)` holding UTC. It rebuilds the table, so writes to `okapi_outbox`, including `publish()`, wait until it finishes; on a large table, purge delivered rows first or apply it in a maintenance window. Apply it before starting updated workers: they read and write the columns as UTC `DATETIME` values. +MySQL's `003` converts the timestamp columns from `TIMESTAMP(6)`, which ends in January 2038, to `DATETIME(6)` holding UTC. Stop every 1.x instance, publishers included, before it runs: 1.x writes these columns in the JVM zone. It copies the table; it first waits (up to 30 s, then fails and retries on the next start) for open transactions on `okapi_outbox` to finish, queuing every other statement on the table meanwhile, and then blocks writes, including `publish()`, until the copy finishes. See [UPGRADING.md](UPGRADING.md#from-1x-to-200), including when rows written by 1.x can read back shifted. okapi stores messages in the fixed `okapi_outbox` table. When the built-in Liquibase integration is used, it also uses two dedicated migration tracking tables: diff --git a/UPGRADING.md b/UPGRADING.md index 9751c655..c573cb09 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -13,8 +13,25 @@ automatically at startup; otherwise apply [`003__datetime_columns.sql`](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql) yourself, **including its `SET time_zone` statements**, before starting 2.0.0 workers. -- The `ALTER TABLE` rebuilds `okapi_outbox` and blocks writes to it, including `publish()`, until - it finishes. Purge delivered rows first or schedule it for a quiet period on a large table. +- **Stop every 1.x instance, publishers included, before 003 runs.** 1.x binds these columns as + `java.sql.Timestamp`, which Connector/J shifts into the JVM (or `connectionTimeZone`) zone. Against + `DATETIME` columns a 1.x instance on a non-UTC JVM writes local wall-clock time instead of UTC and + misreads rows written by 2.0. A rolling deploy that mixes the two is only safe if every 1.x JVM + runs in UTC. +- **Rows written by 1.x may read back shifted if the JVM and MySQL zones differed.** Without + `connectionTimeZone`, 1.x sent `Timestamp`s as JVM-local wall-clock time and MySQL interpreted them + in the session `time_zone`; when the two zones differed, the stored instant was off by the + difference, and 1.x reading it back undid the shift. 003 converts what is actually stored, so 2.0 + reads such rows off by that difference: old `PENDING` rows sort out of order, the lag gauge is skewed + and old `DELIVERED` rows are purged early or late. If your JVMs and MySQL session ran in different + zones, drain the outbox (let every `PENDING` row be delivered) before upgrading. +- **Locking.** The `ALTER TABLE` copies `okapi_outbox` (`ALGORITHM=COPY, LOCK=SHARED`). Before it + starts it waits for a metadata lock behind every open transaction on the table, such as a 1.x worker + delivering a claimed batch, and while it waits every new statement on the table, reads included, + queues behind it. The changeset bounds that wait with `lock_wait_timeout = 30` seconds; on timeout + the migration fails and runs again on the next start. During the copy, writes, including + `publish()`, wait. Stop the workers first, and purge delivered rows or pick a quiet period on a + large table. - Queries or dashboards that read these columns directly now get UTC wall-clock values, not values converted to the session time zone. diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlDatetimeMigrationTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlDatetimeMigrationTest.kt index 6cf29ee7..cb137716 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlDatetimeMigrationTest.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/MysqlDatetimeMigrationTest.kt @@ -41,6 +41,12 @@ class MysqlDatetimeMigrationTest : FunSpec({ test("changeset 003 keeps existing instants and restores the session time zone") { offsetSessionConnection().use { conn -> + val defaultLockWaitTimeout = conn.createStatement().use { stmt -> + stmt.executeQuery("SELECT @@session.lock_wait_timeout").use { rs -> + rs.next() + rs.getLong(1) + } + } val liquibase = Liquibase( CHANGELOG, @@ -63,14 +69,40 @@ class MysqlDatetimeMigrationTest : FunSpec({ liquibase.update("") conn.createStatement().use { stmt -> - stmt.executeQuery( - "SELECT @@session.time_zone AS tz, DATA_TYPE AS t FROM information_schema.COLUMNS " + - "WHERE TABLE_NAME = 'okapi_outbox' AND COLUMN_NAME = 'created_at'", - ).use { rs -> + stmt.executeQuery("SELECT @@session.time_zone AS tz, @@session.lock_wait_timeout AS lwt").use { rs -> rs.next() rs.getString("tz") shouldBe "+05:00" - rs.getString("t") shouldBe "datetime" + rs.getLong("lwt") shouldBe defaultLockWaitTimeout } + val columns = stmt.executeQuery( + """ + SELECT COLUMN_NAME, DATA_TYPE, DATETIME_PRECISION, IS_NULLABLE, COLUMN_DEFAULT, EXTRA + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'okapi_outbox' + AND COLUMN_NAME IN ('created_at', 'updated_at', 'last_attempt') + ORDER BY COLUMN_NAME + """.trimIndent(), + ).use { rs -> + generateSequence { + if (rs.next()) { + listOf( + rs.getString("COLUMN_NAME"), + rs.getString("DATA_TYPE"), + rs.getInt("DATETIME_PRECISION").toString(), + rs.getString("IS_NULLABLE"), + rs.getString("COLUMN_DEFAULT") ?: "null", + rs.getString("EXTRA"), + ).joinToString("|") + } else { + null + } + }.toList() + } + columns shouldBe listOf( + "created_at|datetime|6|NO|null|", + "last_attempt|datetime|6|YES|null|", + "updated_at|datetime|6|NO|null|", + ) } } diff --git a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql index eaae7ba0..2fdc306e 100644 --- a/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql +++ b/okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql @@ -4,15 +4,25 @@ -- TIMESTAMP stops at 2038-01-19 03:14:07 UTC (issue #138). DATETIME(6) has no such limit and, -- unlike TIMESTAMP, is not converted through the session time zone: okapi stores UTC wall-clock -- values in it. MODIFY converts the existing TIMESTAMP values as the session time zone renders --- them, so pin the session to UTC for the conversion and restore it afterwards (Liquibase may run --- on a pooled connection that the application reuses). This rebuilds the table, and writes to it --- (including publish()) wait until the rebuild finishes. +-- them, so pin the session to UTC for the conversion. Liquibase may run on a pooled connection the +-- application reuses, so the session settings are restored afterwards -- on success only: if the +-- ALTER fails, Liquibase aborts startup and the connection keeps the values set here. +-- +-- The ALTER copies the table. It first waits for a metadata lock behind every open transaction that +-- touched okapi_outbox (e.g. a 1.x worker delivering a claimed batch), and while it waits, every new +-- statement on the table queues behind it. lock_wait_timeout bounds that wait to 30 s instead of +-- MySQL's default of a year; on timeout the migration fails and runs again on the next start. Once +-- the copy starts, writes to okapi_outbox (including publish()) wait until it finishes. SET @okapi_previous_time_zone = @@session.time_zone; +SET @okapi_previous_lock_wait_timeout = @@session.lock_wait_timeout; SET time_zone = '+00:00'; +SET SESSION lock_wait_timeout = 30; ALTER TABLE okapi_outbox MODIFY created_at DATETIME(6) NOT NULL, MODIFY updated_at DATETIME(6) NOT NULL, - MODIFY last_attempt DATETIME(6) NULL; + MODIFY last_attempt DATETIME(6) NULL, + ALGORITHM=COPY, LOCK=SHARED; +SET SESSION lock_wait_timeout = @okapi_previous_lock_wait_timeout; SET time_zone = @okapi_previous_time_zone;