From 9f4e5801b2ab9b16f4f33262c625d18cd710e217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Wokacz?= Date: Thu, 5 Nov 2026 17:02:11 +0100 Subject: [PATCH 1/2] fix(postgres)!: store timestamps as TIMESTAMPTZ so they no longer depend on the JVM zone (#135) created_at, updated_at and last_attempt were TIMESTAMP WITHOUT TIME ZONE, and pgjdbc renders a java.sql.Timestamp in the JVM default zone, so each row held the writing JVM's local wall-clock time; mixed-zone fleets and DST fall-back skewed claim order, lag and purge by hours. Changeset 003 converts the columns to TIMESTAMPTZ, reading existing values in the session TimeZone, which pgjdbc sets to the JVM default zone, so the bundled changelog run by the application keeps them correct. lock_timeout bounds its ACCESS EXCLUSIVE lock wait to 30 s. PostgresOutboxStore binds and reads OffsetDateTime. BREAKING CHANGE: new migration 003 rewrites okapi_outbox on PostgreSQL under an ACCESS EXCLUSIVE lock; applied by hand it needs SET TIME ZONE to the okapi JVM zone first. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 ++ README.md | 4 +- UPGRADING.md | 26 +++++ .../store/PostgresTimestampColumnsTest.kt | 20 ++++ .../store/PostgresTimestamptzMigrationTest.kt | 107 ++++++++++++++++++ .../okapi/postgres/PostgresOutboxStore.kt | 50 ++++---- .../db/postgres/003__timestamptz_columns.sql | 24 ++++ .../okapi/db/postgres/changelog.xml | 1 + 8 files changed, 218 insertions(+), 26 deletions(-) create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestampColumnsTest.kt create mode 100644 okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestamptzMigrationTest.kt create mode 100644 okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c63f2..b9c6026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -152,6 +152,18 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * copies. Anything else reading these columns directly must now treat them as UTC wall-clock values. See UPGRADING.md. ([#138](https://github.com/softwaremill/okapi/issues/138)) +- **PostgreSQL: timestamp columns are `TIMESTAMPTZ` (new migration `003__timestamptz_columns`).** + `created_at`, `updated_at` and `last_attempt` were `TIMESTAMP WITHOUT TIME ZONE`, and pgjdbc + renders a `java.sql.Timestamp` in the JVM default zone, so each row held the writing JVM's local + wall-clock time. In a fleet whose JVMs ran in different zones, or across a DST fall-back hour, + claim ordering, the lag gauge and the purger's retention cutoff were off by hours. The migration + converts existing values using the session `TimeZone`, which pgjdbc sets to the JVM default zone, + so the bundled Liquibase changelog keeps them correct when it runs inside the application; apply + it by hand only after `SET TIME ZONE ''`. `PostgresOutboxStore` now binds and reads + these columns as `OffsetDateTime`. The migration takes an `ACCESS EXCLUSIVE` lock (waiting at most + 30 s for it, then failing and retrying on the next start) and, unless the session zone is UTC, + rewrites the table. 1.x instances keep working against the migrated columns. Rows written during a fall-back hour by a non-UTC JVM were + already ambiguous and may stay an hour off. ([#135](https://github.com/softwaremill/okapi/issues/135)) ## [1.0.0] — 2026-07-28 diff --git a/README.md b/README.md index 91307e6..852fd8c 100644 --- a/README.md +++ b/README.md @@ -215,11 +215,13 @@ With `okapi-spring-boot` and Liquibase on the classpath, these run automatically If you use another migration tool, apply every SQL migration for your database, in order: -- **PostgreSQL:** [001](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql) route-aware claim index +- **PostgreSQL:** [001](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/002__route_aware_claim_index.sql) route-aware claim index, [003](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql) `TIMESTAMPTZ` timestamp columns - **MySQL:** [001](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/001__create_okapi_outbox_table.sql) table and initial indexes, [002](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/002__route_aware_claim_index.sql) route-aware claim index, [003](okapi-mysql/src/main/resources/com/softwaremill/okapi/db/mysql/003__datetime_columns.sql) `DATETIME(6)` timestamp columns For an existing installation, apply `002` before starting updated workers. MySQL claims explicitly select the new index and fail if it is missing. PostgreSQL's `002` uses `CREATE INDEX CONCURRENTLY`, so run it outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. `CREATE INDEX CONCURRENTLY` waits for every transaction in the database that started before it, not only those on `okapi_outbox`, so long-running transactions (including old workers delivering a claimed batch) delay it, and with Liquibase they delay application startup while the changelog lock is held. Run it when no long transactions are open, or apply it ahead of the rollout. +PostgreSQL's `003` converts the timestamp columns from `TIMESTAMP` (the okapi JVM's local wall-clock time) to `TIMESTAMPTZ`. It reads existing values in the session `TimeZone`; pgjdbc sets that to the JVM default zone, so the bundled Liquibase changelog converts them correctly when it runs inside the application. When applying it by hand, run `SET TIME ZONE ''` first in the same session. It takes an `ACCESS EXCLUSIVE` lock: it first waits (up to 30 s, then fails and retries on the next start) for open transactions on `okapi_outbox`, queuing every other statement on the table meanwhile, and then, unless the session zone is UTC, rewrites the table while reads and writes wait. 1.x and 2.0 instances can run side by side after it. See [UPGRADING.md](UPGRADING.md#from-1x-to-200). + MySQL's `003` converts the timestamp columns from `TIMESTAMP(6)`, which ends in January 2038, to `DATETIME(6)` holding UTC. Stop every 1.x instance, publishers included, before it runs: 1.x writes these columns in the JVM zone. It copies the table; it first waits (up to 30 s, then fails and retries on the next start) for open transactions on `okapi_outbox` to finish, queuing every other statement on the table meanwhile, and then blocks writes, including `publish()`, until the copy finishes. See [UPGRADING.md](UPGRADING.md#from-1x-to-200), including when rows written by 1.x can read back shifted. okapi stores messages in the fixed `okapi_outbox` table. When the built-in Liquibase integration is used, it also uses two dedicated migration tracking tables: diff --git a/UPGRADING.md b/UPGRADING.md index c573cb0..c880252 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -35,6 +35,32 @@ yourself, **including its `SET time_zone` statements**, before starting 2.0.0 wo - Queries or dashboards that read these columns directly now get UTC wall-clock values, not values converted to the session time zone. +### PostgreSQL: `TIMESTAMPTZ` timestamp columns (#135) + +Changeset `003__timestamptz_columns` converts `created_at`, `updated_at` and `last_attempt` from +`TIMESTAMP` to `TIMESTAMPTZ`. Existing values hold the local wall-clock time of the JVM that wrote +them, and the conversion reads them in the session `TimeZone`: + +- With the bundled Liquibase changelog running inside your application, pgjdbc sets the session zone + to the JVM default zone, so nothing extra is needed as long as the migrating JVM runs in the same + zone as the JVMs that wrote the rows. +- Applying + [`003__timestamptz_columns.sql`](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql) + yourself: run `SET TIME ZONE '';` first, in the same session (e.g. + `SET TIME ZONE 'UTC';` for JVMs running in UTC). +- If your JVMs ran in different zones, rows written by each were stored in that JVM's zone; no single + conversion fixes all of them. Drain the outbox (let every `PENDING` row be delivered) before + migrating if accurate history matters. +- **Locking.** The `ALTER TABLE` needs an `ACCESS EXCLUSIVE` lock. It first waits behind every open + transaction on `okapi_outbox`, such as a worker delivering a claimed batch, and every new statement + on the table, reads included, queues behind it meanwhile. The changeset bounds that wait with + `SET LOCAL lock_timeout = '30s'`; on timeout the migration fails and runs again on the next start. + Holding the lock, it rewrites the table unless the session zone is UTC, and reads and writes, + including `publish()`, wait until it finishes. Purge delivered rows or pick a quiet period on a + large table. +- **Rolling deploys are safe.** 1.x binds `java.sql.Timestamp`, which pgjdbc sends with the JVM's + offset, so 1.x instances write and read `TIMESTAMPTZ` columns correctly after the migration. + ## From 0.2.x to 1.0.0 Releases up to 0.2.x used the domain table `outbox` and the shared Liquibase tracking diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestampColumnsTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestampColumnsTest.kt new file mode 100644 index 0000000..6a507a6 --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestampColumnsTest.kt @@ -0,0 +1,20 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.postgres.PostgresOutboxStore +import com.softwaremill.okapi.test.support.PostgresTestSupport +import io.kotest.core.spec.style.FunSpec + +class PostgresTimestampColumnsTest : FunSpec({ + val db = PostgresTestSupport() + + timestampColumnTests( + dbName = "postgres", + jdbcProvider = { db.jdbc }, + storeFactory = { PostgresOutboxStore(db.jdbc) }, + startDb = { db.start() }, + stopDb = { db.stop() }, + truncate = { db.truncate() }, + utcText = { column -> "to_char($column AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US')" }, + setSessionZone = "SET TIME ZONE 'Asia/Kolkata'", + ) +}) diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestamptzMigrationTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestamptzMigrationTest.kt new file mode 100644 index 0000000..fba1de6 --- /dev/null +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestamptzMigrationTest.kt @@ -0,0 +1,107 @@ +package com.softwaremill.okapi.test.store + +import com.softwaremill.okapi.postgres.PostgresOutboxStore +import com.softwaremill.okapi.test.support.JdbcConnectionProvider +import com.softwaremill.okapi.test.support.pgDataSourceOf +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import liquibase.Liquibase +import liquibase.database.DatabaseFactory +import liquibase.database.jvm.JdbcConnection +import liquibase.resource.ClassLoaderResourceAccessor +import org.testcontainers.containers.PostgreSQLContainer +import java.sql.DriverManager +import java.sql.Timestamp +import java.time.Instant +import java.util.TimeZone + +private const val CHANGELOG = "com/softwaremill/okapi/db/postgres/changelog.xml" + +/** + * Upgrade path for issue #135: before changeset 003, pgjdbc stored a `java.sql.Timestamp` as the + * JVM zone's wall-clock time in TIMESTAMP WITHOUT TIME ZONE. Run inside that same JVM, 003 must + * turn those values back into the instants that were written, because pgjdbc sets the session + * TimeZone to the JVM default zone and the type change reads values in the session zone. + */ +class PostgresTimestamptzMigrationTest : FunSpec({ + val container = PostgreSQLContainer("postgres:16") + + beforeSpec { container.start() } + afterSpec { container.stop() } + + test("changeset 003 keeps the instants written by a non-UTC JVM") { + val createdAt = Instant.parse("2024-10-27T01:30:00.123456Z") + val lastAttempt = Instant.parse("2024-06-01T12:00:00Z") + val original = TimeZone.getDefault() + try { + TimeZone.setDefault(TimeZone.getTimeZone("Asia/Tokyo")) + DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { conn -> + val liquibase = + Liquibase( + CHANGELOG, + ClassLoaderResourceAccessor(), + DatabaseFactory.getInstance().findCorrectDatabaseImplementation(JdbcConnection(conn)), + ) + liquibase.update(2, "") + + // Exactly how 1.x PostgresOutboxStore wrote these columns. + conn.prepareStatement( + """ + INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, delivery_metadata) + VALUES ('00000000-0000-0000-0000-000000000001', 'm', '{}', 'http', 'PENDING', ?, ?, 1, ?, '{}'::jsonb) + """.trimIndent(), + ).use { stmt -> + stmt.setTimestamp(1, Timestamp.from(createdAt)) + stmt.setTimestamp(2, Timestamp.from(createdAt)) + stmt.setTimestamp(3, Timestamp.from(lastAttempt)) + stmt.executeUpdate() + } + + liquibase.update("") + + conn.createStatement().use { stmt -> + stmt.executeQuery( + "SELECT data_type FROM information_schema.columns WHERE table_name = 'okapi_outbox' AND column_name = 'created_at'", + ).use { rs -> + rs.next() + rs.getString("data_type") shouldBe "timestamp with time zone" + } + } + } + } finally { + TimeZone.setDefault(original) + } + + val jdbc = JdbcConnectionProvider(pgDataSourceOf(container)) + val entry = jdbc.withTransaction { PostgresOutboxStore(jdbc).claimPending(10) }.single() + entry.createdAt shouldBe createdAt + entry.updatedAt shouldBe createdAt + entry.lastAttempt shouldBe lastAttempt + } + + test("a 1.x instance on a non-UTC JVM still writes correct instants after changeset 003") { + val createdAt = Instant.parse("2025-01-15T08:00:00.5Z") + val original = TimeZone.getDefault() + try { + TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")) + DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { conn -> + // Exactly how 1.x PostgresOutboxStore wrote these columns. + conn.prepareStatement( + """ + INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, delivery_metadata) + VALUES ('00000000-0000-0000-0000-000000000002', 'm', '{}', 'kafka', 'PENDING', ?, ?, 0, '{}'::jsonb) + """.trimIndent(), + ).use { stmt -> + stmt.setTimestamp(1, Timestamp.from(createdAt)) + stmt.setTimestamp(2, Timestamp.from(createdAt)) + stmt.executeUpdate() + } + } + } finally { + TimeZone.setDefault(original) + } + + val jdbc = JdbcConnectionProvider(pgDataSourceOf(container)) + jdbc.withTransaction { PostgresOutboxStore(jdbc).claimPending(setOf("kafka"), 10) }.single().createdAt shouldBe createdAt + } +}) diff --git a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt index 4fce105..b940974 100644 --- a/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt +++ b/okapi-postgres/src/main/kotlin/com/softwaremill/okapi/postgres/PostgresOutboxStore.kt @@ -7,9 +7,10 @@ import com.softwaremill.okapi.core.OutboxStatus import com.softwaremill.okapi.core.RouteAwareOutboxStore import java.sql.PreparedStatement import java.sql.ResultSet -import java.sql.Timestamp import java.sql.Types import java.time.Instant +import java.time.OffsetDateTime +import java.time.ZoneOffset import java.util.UUID /** PostgreSQL [OutboxStore] implementation using plain JDBC. */ @@ -25,17 +26,10 @@ class PostgresOutboxStore( stmt.setString(3, entry.payload) stmt.setString(4, entry.deliveryType) stmt.setString(5, entry.status.name) - stmt.setTimestamp(6, Timestamp.from(entry.createdAt)) - stmt.setTimestamp(7, Timestamp.from(entry.updatedAt)) + stmt.setInstant(6, entry.createdAt) + stmt.setInstant(7, entry.updatedAt) stmt.setInt(8, entry.retries) - if (entry.lastAttempt != null) { - stmt.setTimestamp( - 9, - Timestamp.from(entry.lastAttempt), - ) - } else { - stmt.setNull(9, Types.TIMESTAMP) - } + stmt.setInstant(9, entry.lastAttempt) val lastError = entry.lastError if (lastError != null) stmt.setString(10, OutboxEntry.sanitizeLastError(lastError)) else stmt.setNull(10, Types.VARCHAR) stmt.setString(11, entry.deliveryMetadata) @@ -113,7 +107,7 @@ class PostgresOutboxStore( return connectionProvider.withConnection { conn -> conn.prepareStatement(sql).use { stmt -> stmt.setString(1, OutboxStatus.DELIVERED.name) - stmt.setTimestamp(2, Timestamp.from(time)) + stmt.setInstant(2, time) stmt.setInt(3, limit) stmt.executeUpdate() } @@ -133,7 +127,7 @@ class PostgresOutboxStore( stmt.executeQuery().use { rs -> while (rs.next()) { val s = OutboxStatus.from(rs.getString("status")) - result[s] = rs.getTimestamp("min_created_at").toInstant() + result[s] = checkNotNull(rs.getInstant("min_created_at")) } } } @@ -176,32 +170,38 @@ class PostgresOutboxStore( payload = getString("payload"), deliveryType = getString("delivery_type"), status = OutboxStatus.from(getString("status")), - createdAt = getTimestamp("created_at").toInstant(), - updatedAt = getTimestamp("updated_at").toInstant(), + createdAt = checkNotNull(getInstant("created_at")), + updatedAt = checkNotNull(getInstant("updated_at")), retries = getInt("retries"), - lastAttempt = getTimestamp("last_attempt")?.toInstant(), + lastAttempt = getInstant("last_attempt"), lastError = getString("last_error"), deliveryMetadata = getString("delivery_metadata"), ) private fun PreparedStatement.prepareForUpdateFrom(entry: OutboxEntry): PreparedStatement { setString(1, entry.status.name) - setTimestamp(2, Timestamp.from(entry.updatedAt)) + setInstant(2, entry.updatedAt) setInt(3, entry.retries) - if (entry.lastAttempt != null) { - setTimestamp( - 4, - Timestamp.from(entry.lastAttempt), - ) - } else { - setNull(4, Types.TIMESTAMP) - } + setInstant(4, entry.lastAttempt) val lastError = entry.lastError if (lastError != null) setString(5, OutboxEntry.sanitizeLastError(lastError)) else setNull(5, Types.VARCHAR) setObject(6, entry.outboxId.raw) return this } + // The TIMESTAMPTZ columns hold instants (issue #135). pgjdbc binds an OffsetDateTime as timestamptz + // with its offset, unlike java.sql.Timestamp, which it renders as the JVM zone's wall-clock time -- + // so stored values no longer depend on the JVM or session time zone. + private fun PreparedStatement.setInstant(index: Int, instant: Instant?) { + if (instant != null) { + setObject(index, OffsetDateTime.ofInstant(instant, ZoneOffset.UTC)) + } else { + setNull(index, Types.TIMESTAMP_WITH_TIMEZONE) + } + } + + private fun ResultSet.getInstant(column: String): Instant? = getObject(column, OffsetDateTime::class.java)?.toInstant() + companion object { private val insertSql = """ INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, last_error, delivery_metadata) diff --git a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql new file mode 100644 index 0000000..c0224cc --- /dev/null +++ b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql @@ -0,0 +1,24 @@ +--liquibase formatted sql +--changeset outbox:003 + +-- TIMESTAMP WITHOUT TIME ZONE stored the writer JVM's local wall-clock time (pgjdbc renders a +-- java.sql.Timestamp in the JVM default zone), so ordering, lag and purge broke across mixed-zone +-- fleets and DST fall-back (issue #135). TIMESTAMPTZ stores instants. +-- +-- Changing the type interprets each existing value in the session TimeZone. pgjdbc sets that to the +-- JVM default zone, so when the bundled Liquibase changelog runs inside the application it reads the +-- values in the same zone the application wrote them in. When applying this script by hand, first run +-- SET TIME ZONE ''; in the same session. +-- +-- The ALTER needs an ACCESS EXCLUSIVE lock. It first waits behind every open transaction on +-- okapi_outbox (e.g. a worker delivering a claimed batch), and while it waits every new statement on +-- the table queues behind it; SET LOCAL lock_timeout bounds that wait to 30 s for this transaction +-- only, after which the migration fails and runs again on the next start. Holding the lock, it +-- rewrites the table unless the session zone is UTC and rebuilds its indexes; reads and writes to +-- okapi_outbox, including publish(), wait until it finishes. +SET LOCAL lock_timeout = '30s'; + +ALTER TABLE okapi_outbox + ALTER COLUMN created_at TYPE TIMESTAMPTZ, + ALTER COLUMN updated_at TYPE TIMESTAMPTZ, + ALTER COLUMN last_attempt TYPE TIMESTAMPTZ; diff --git a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/changelog.xml b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/changelog.xml index 571916b..bbc4f37 100644 --- a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/changelog.xml +++ b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/changelog.xml @@ -5,4 +5,5 @@ http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd"> + From d4b16d2ad2f692384b47f0f4cd7d2ae08e78fecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Wokacz?= Date: Fri, 6 Nov 2026 09:58:19 +0100 Subject: [PATCH 2/2] fix(postgres): address review of the TIMESTAMPTZ migration (#135) - Document that a manual run needs a transaction (SET LOCAL is a no-op outside one), the cases where the in-app conversion uses the wrong zone, PostgreSQL 12+ for skipping the rewrite, and what really happens to 1.x pooled connections (prepared writes stay correct, a pre-prepared claim fails once). - Migration tests use a database each, close Liquibase, and cover a 1.x connection with statements server-prepared before 003 plus the 1.x read path. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 12 +- README.md | 2 +- UPGRADING.md | 19 +- .../store/PostgresTimestamptzMigrationTest.kt | 168 ++++++++++++------ .../db/postgres/003__timestamptz_columns.sql | 7 +- 5 files changed, 139 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9c6026..24ebbb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -159,11 +159,13 @@ Until `1.0.0`, breaking changes may appear in any release and are flagged with * claim ordering, the lag gauge and the purger's retention cutoff were off by hours. The migration converts existing values using the session `TimeZone`, which pgjdbc sets to the JVM default zone, so the bundled Liquibase changelog keeps them correct when it runs inside the application; apply - it by hand only after `SET TIME ZONE ''`. `PostgresOutboxStore` now binds and reads - these columns as `OffsetDateTime`. The migration takes an `ACCESS EXCLUSIVE` lock (waiting at most - 30 s for it, then failing and retrying on the next start) and, unless the session zone is UTC, - rewrites the table. 1.x instances keep working against the migrated columns. Rows written during a fall-back hour by a non-UTC JVM were - already ambiguous and may stay an hour off. ([#135](https://github.com/softwaremill/okapi/issues/135)) + it by hand only inside a transaction and after `SET TIME ZONE ''`. + `PostgresOutboxStore` now binds and reads these columns as `OffsetDateTime`. The migration takes an + `ACCESS EXCLUSIVE` lock (waiting at most 30 s for it, then failing and retrying on the next start) + and, unless the session zone is UTC, rewrites the table. 1.x instances keep working against the + migrated columns, apart from one failed poll per pooled connection. Rows written during a + fall-back hour by a non-UTC JVM were already ambiguous and may stay an hour off. See UPGRADING.md. + ([#135](https://github.com/softwaremill/okapi/issues/135)) ## [1.0.0] — 2026-07-28 diff --git a/README.md b/README.md index 852fd8c..2981ead 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ If you use another migration tool, apply every SQL migration for your database, For an existing installation, apply `002` before starting updated workers. MySQL claims explicitly select the new index and fail if it is missing. PostgreSQL's `002` uses `CREATE INDEX CONCURRENTLY`, so run it outside a transaction when using another migration tool. The bundled Liquibase changelog handles this automatically. `CREATE INDEX CONCURRENTLY` waits for every transaction in the database that started before it, not only those on `okapi_outbox`, so long-running transactions (including old workers delivering a claimed batch) delay it, and with Liquibase they delay application startup while the changelog lock is held. Run it when no long transactions are open, or apply it ahead of the rollout. -PostgreSQL's `003` converts the timestamp columns from `TIMESTAMP` (the okapi JVM's local wall-clock time) to `TIMESTAMPTZ`. It reads existing values in the session `TimeZone`; pgjdbc sets that to the JVM default zone, so the bundled Liquibase changelog converts them correctly when it runs inside the application. When applying it by hand, run `SET TIME ZONE ''` first in the same session. It takes an `ACCESS EXCLUSIVE` lock: it first waits (up to 30 s, then fails and retries on the next start) for open transactions on `okapi_outbox`, queuing every other statement on the table meanwhile, and then, unless the session zone is UTC, rewrites the table while reads and writes wait. 1.x and 2.0 instances can run side by side after it. See [UPGRADING.md](UPGRADING.md#from-1x-to-200). +PostgreSQL's `003` converts the timestamp columns from `TIMESTAMP` (the okapi JVM's local wall-clock time) to `TIMESTAMPTZ`. It reads existing values in the session `TimeZone`; pgjdbc sets that to the JVM default zone, so the bundled Liquibase changelog converts them correctly when it runs inside the application. When applying it by hand, run it inside a transaction (`psql -1`) after `SET TIME ZONE ''` in the same session. It takes an `ACCESS EXCLUSIVE` lock: it first waits (up to 30 s, then fails and retries on the next start) for open transactions on `okapi_outbox`, queuing every other statement on the table meanwhile, and then, unless the session zone is UTC (PostgreSQL 12+), rewrites the table while reads and writes wait. 1.x instances keep working after it, apart from one failed poll per pooled connection. See UPGRADING.md for when the in-app conversion uses the wrong zone. See [UPGRADING.md](UPGRADING.md#from-1x-to-200). MySQL's `003` converts the timestamp columns from `TIMESTAMP(6)`, which ends in January 2038, to `DATETIME(6)` holding UTC. Stop every 1.x instance, publishers included, before it runs: 1.x writes these columns in the JVM zone. It copies the table; it first waits (up to 30 s, then fails and retries on the next start) for open transactions on `okapi_outbox` to finish, queuing every other statement on the table meanwhile, and then blocks writes, including `publish()`, until the copy finishes. See [UPGRADING.md](UPGRADING.md#from-1x-to-200), including when rows written by 1.x can read back shifted. diff --git a/UPGRADING.md b/UPGRADING.md index c880252..b9b50c3 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -46,8 +46,14 @@ them, and the conversion reads them in the session `TimeZone`: zone as the JVMs that wrote the rows. - Applying [`003__timestamptz_columns.sql`](okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql) - yourself: run `SET TIME ZONE '';` first, in the same session (e.g. - `SET TIME ZONE 'UTC';` for JVMs running in UTC). + yourself: run it inside a transaction (`BEGIN` … `COMMIT`, or `psql -1`; outside one its + `SET LOCAL lock_timeout` has no effect), and run `SET TIME ZONE '';` + first in the same session (e.g. `SET TIME ZONE 'UTC';` for JVMs running in UTC). +- The in-app conversion uses the zone pgjdbc gave the session when the connection was opened. It is + wrong if that differs from the zone your instances wrote with: if the application calls + `TimeZone.setDefault` after its connection pool has opened connections, if a pool init SQL (e.g. + HikariCP `connectionInitSql`) or the pgjdbc `options` property changes the session `TimeZone`, or if + the migrating JVM runs in another zone than the writers. In those cases apply 003 by hand as above. - If your JVMs ran in different zones, rows written by each were stored in that JVM's zone; no single conversion fixes all of them. Drain the outbox (let every `PENDING` row be delivered) before migrating if accurate history matters. @@ -58,8 +64,13 @@ them, and the conversion reads them in the session `TimeZone`: Holding the lock, it rewrites the table unless the session zone is UTC, and reads and writes, including `publish()`, wait until it finishes. Purge delivered rows or pick a quiet period on a large table. -- **Rolling deploys are safe.** 1.x binds `java.sql.Timestamp`, which pgjdbc sends with the JVM's - offset, so 1.x instances write and read `TIMESTAMPTZ` columns correctly after the migration. +- **Rolling deploys work, with one transient error per pooled 1.x connection.** 1.x binds + `java.sql.Timestamp`, which pgjdbc sends with the JVM's offset. After the migration, a fresh 1.x + statement stores it as the right instant, and a statement server-prepared before the migration + keeps its `timestamp` parameter type, so the offset is dropped and the value is read in the session + zone, which pgjdbc set to that same JVM zone: also the right instant. Reads via `getTimestamp` are + correct too. A claim query a 1.x connection prepared before the migration fails once with + `cached plan must not change result type`; that tick is retried on the next poll. ## From 0.2.x to 1.0.0 diff --git a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestamptzMigrationTest.kt b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestamptzMigrationTest.kt index fba1de6..e00f732 100644 --- a/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestamptzMigrationTest.kt +++ b/okapi-integration-tests/src/test/kotlin/com/softwaremill/okapi/test/store/PostgresTimestamptzMigrationTest.kt @@ -2,26 +2,35 @@ package com.softwaremill.okapi.test.store import com.softwaremill.okapi.postgres.PostgresOutboxStore import com.softwaremill.okapi.test.support.JdbcConnectionProvider -import com.softwaremill.okapi.test.support.pgDataSourceOf import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.shouldBe import liquibase.Liquibase import liquibase.database.DatabaseFactory import liquibase.database.jvm.JdbcConnection import liquibase.resource.ClassLoaderResourceAccessor +import org.postgresql.ds.PGSimpleDataSource import org.testcontainers.containers.PostgreSQLContainer +import java.sql.Connection import java.sql.DriverManager import java.sql.Timestamp import java.time.Instant +import java.util.Properties import java.util.TimeZone private const val CHANGELOG = "com/softwaremill/okapi/db/postgres/changelog.xml" +private const val INSERT_1X = """ + INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, delivery_metadata) + VALUES (?::uuid, 'm', '{}', 'http', 'PENDING', ?, ?, 1, ?, '{}'::jsonb) +""" + /** * Upgrade path for issue #135: before changeset 003, pgjdbc stored a `java.sql.Timestamp` as the * JVM zone's wall-clock time in TIMESTAMP WITHOUT TIME ZONE. Run inside that same JVM, 003 must * turn those values back into the instants that were written, because pgjdbc sets the session * TimeZone to the JVM default zone and the type change reads values in the session zone. + * + * Each test migrates its own database, so they do not depend on each other or on run order. */ class PostgresTimestamptzMigrationTest : FunSpec({ val container = PostgreSQLContainer("postgres:16") @@ -29,79 +38,126 @@ class PostgresTimestamptzMigrationTest : FunSpec({ beforeSpec { container.start() } afterSpec { container.stop() } + /** Creates an empty database [name] and returns its JDBC URL. */ + fun freshDatabase(name: String): String { + DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { conn -> + conn.createStatement().use { it.execute("CREATE DATABASE $name") } + } + return container.jdbcUrl.replaceAfterLast('/', name) + } + + fun connect(url: String, extra: Properties.() -> Unit = {}): Connection = DriverManager.getConnection( + url, + Properties().apply { + setProperty("user", container.username) + setProperty("password", container.password) + extra() + }, + ) + + /** Runs the first [changes] changesets (all of them when null) on a connection of its own. */ + fun migrate(url: String, changes: Int? = null) { + connect(url).use { conn -> + Liquibase( + CHANGELOG, + ClassLoaderResourceAccessor(), + DatabaseFactory.getInstance().findCorrectDatabaseImplementation(JdbcConnection(conn)), + ) + .use { if (changes != null) it.update(changes, "") else it.update("") } + } + } + + fun store(url: String): Pair { + val jdbc = JdbcConnectionProvider( + PGSimpleDataSource().apply { + setURL(url) + user = container.username + password = container.password + }, + ) + return jdbc to PostgresOutboxStore(jdbc) + } + + fun inJvmZone(zone: String, block: () -> T): T { + val original = TimeZone.getDefault() + TimeZone.setDefault(TimeZone.getTimeZone(zone)) + return try { + block() + } finally { + TimeZone.setDefault(original) + } + } + test("changeset 003 keeps the instants written by a non-UTC JVM") { + val url = freshDatabase("upgrade_path") val createdAt = Instant.parse("2024-10-27T01:30:00.123456Z") val lastAttempt = Instant.parse("2024-06-01T12:00:00Z") - val original = TimeZone.getDefault() - try { - TimeZone.setDefault(TimeZone.getTimeZone("Asia/Tokyo")) - DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { conn -> - val liquibase = - Liquibase( - CHANGELOG, - ClassLoaderResourceAccessor(), - DatabaseFactory.getInstance().findCorrectDatabaseImplementation(JdbcConnection(conn)), - ) - liquibase.update(2, "") - - // Exactly how 1.x PostgresOutboxStore wrote these columns. - conn.prepareStatement( - """ - INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, last_attempt, delivery_metadata) - VALUES ('00000000-0000-0000-0000-000000000001', 'm', '{}', 'http', 'PENDING', ?, ?, 1, ?, '{}'::jsonb) - """.trimIndent(), - ).use { stmt -> - stmt.setTimestamp(1, Timestamp.from(createdAt)) + + inJvmZone("Asia/Tokyo") { + migrate(url, changes = 2) + // Exactly how 1.x PostgresOutboxStore wrote these columns. + connect(url).use { conn -> + conn.prepareStatement(INSERT_1X.trimIndent()).use { stmt -> + stmt.setString(1, "00000000-0000-0000-0000-000000000001") stmt.setTimestamp(2, Timestamp.from(createdAt)) - stmt.setTimestamp(3, Timestamp.from(lastAttempt)) + stmt.setTimestamp(3, Timestamp.from(createdAt)) + stmt.setTimestamp(4, Timestamp.from(lastAttempt)) stmt.executeUpdate() } + } + migrate(url) + } - liquibase.update("") - - conn.createStatement().use { stmt -> - stmt.executeQuery( - "SELECT data_type FROM information_schema.columns WHERE table_name = 'okapi_outbox' AND column_name = 'created_at'", - ).use { rs -> - rs.next() - rs.getString("data_type") shouldBe "timestamp with time zone" - } + connect(url).use { conn -> + conn.createStatement().use { stmt -> + stmt.executeQuery( + "SELECT data_type FROM information_schema.columns WHERE table_name = 'okapi_outbox' AND column_name = 'created_at'", + ).use { rs -> + rs.next() + rs.getString("data_type") shouldBe "timestamp with time zone" } } - } finally { - TimeZone.setDefault(original) } - - val jdbc = JdbcConnectionProvider(pgDataSourceOf(container)) - val entry = jdbc.withTransaction { PostgresOutboxStore(jdbc).claimPending(10) }.single() + val (jdbc, store) = store(url) + val entry = jdbc.withTransaction { store.claimPending(10) }.single() entry.createdAt shouldBe createdAt entry.updatedAt shouldBe createdAt entry.lastAttempt shouldBe lastAttempt } - test("a 1.x instance on a non-UTC JVM still writes correct instants after changeset 003") { - val createdAt = Instant.parse("2025-01-15T08:00:00.5Z") - val original = TimeZone.getDefault() - try { - TimeZone.setDefault(TimeZone.getTimeZone("America/New_York")) - DriverManager.getConnection(container.jdbcUrl, container.username, container.password).use { conn -> - // Exactly how 1.x PostgresOutboxStore wrote these columns. - conn.prepareStatement( - """ - INSERT INTO okapi_outbox (id, message_type, payload, delivery_type, status, created_at, updated_at, retries, delivery_metadata) - VALUES ('00000000-0000-0000-0000-000000000002', 'm', '{}', 'kafka', 'PENDING', ?, ?, 0, '{}'::jsonb) - """.trimIndent(), - ).use { stmt -> - stmt.setTimestamp(1, Timestamp.from(createdAt)) - stmt.setTimestamp(2, Timestamp.from(createdAt)) - stmt.executeUpdate() + test("a 1.x connection whose statements were server-prepared before 003 still writes and reads correct instants") { + val url = freshDatabase("rolling_deploy") + val before = Instant.parse("2025-01-15T08:00:00.5Z") + val after = Instant.parse("2025-01-15T09:00:00.25Z") + + inJvmZone("America/New_York") { + migrate(url, changes = 2) + // prepareThreshold=1: pgjdbc server-prepares on first use, as a long-lived pooled 1.x + // connection will have done long before the migration. + connect(url) { setProperty("prepareThreshold", "1") }.use { conn -> + conn.prepareStatement(INSERT_1X.trimIndent()).use { insert -> + fun write(id: String, at: Instant) { + insert.setString(1, id) + insert.setTimestamp(2, Timestamp.from(at)) + insert.setTimestamp(3, Timestamp.from(at)) + insert.setTimestamp(4, Timestamp.from(at)) + insert.executeUpdate() + } + write("00000000-0000-0000-0000-000000000002", before) + migrate(url) + // Same prepared statement, now against TIMESTAMPTZ columns. + write("00000000-0000-0000-0000-000000000003", after) + } + // 1.x read path: getTimestamp on a statement prepared after the migration. + conn.prepareStatement("SELECT created_at FROM okapi_outbox ORDER BY created_at").use { select -> + select.executeQuery().use { rs -> + generateSequence { if (rs.next()) rs.getTimestamp("created_at").toInstant() else null }.toList() + } shouldBe listOf(before, after) } } - } finally { - TimeZone.setDefault(original) } - val jdbc = JdbcConnectionProvider(pgDataSourceOf(container)) - jdbc.withTransaction { PostgresOutboxStore(jdbc).claimPending(setOf("kafka"), 10) }.single().createdAt shouldBe createdAt + val (jdbc, store) = store(url) + jdbc.withTransaction { store.claimPending(10) }.map { it.createdAt } shouldBe listOf(before, after) } }) diff --git a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql index c0224cc..1005430 100644 --- a/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql +++ b/okapi-postgres/src/main/resources/com/softwaremill/okapi/db/postgres/003__timestamptz_columns.sql @@ -7,14 +7,15 @@ -- -- Changing the type interprets each existing value in the session TimeZone. pgjdbc sets that to the -- JVM default zone, so when the bundled Liquibase changelog runs inside the application it reads the --- values in the same zone the application wrote them in. When applying this script by hand, first run --- SET TIME ZONE ''; in the same session. +-- values in the same zone the application wrote them in. When applying this script by hand, run it +-- inside a transaction (BEGIN ... COMMIT, or psql -1; outside one, SET LOCAL below has no effect) and +-- first run SET TIME ZONE ''; in the same session. -- -- The ALTER needs an ACCESS EXCLUSIVE lock. It first waits behind every open transaction on -- okapi_outbox (e.g. a worker delivering a claimed batch), and while it waits every new statement on -- the table queues behind it; SET LOCAL lock_timeout bounds that wait to 30 s for this transaction -- only, after which the migration fails and runs again on the next start. Holding the lock, it --- rewrites the table unless the session zone is UTC and rebuilds its indexes; reads and writes to +-- rewrites the table unless the session zone is UTC (PostgreSQL 12+) and rebuilds its indexes; reads and writes to -- okapi_outbox, including publish(), wait until it finishes. SET LOCAL lock_timeout = '30s';