diff --git a/server/api/store/pg/migrations/020_drop_device_presence_indexes.tx.down.sql b/server/api/store/pg/migrations/020_drop_device_presence_indexes.tx.down.sql new file mode 100644 index 00000000000..077b83557c6 --- /dev/null +++ b/server/api/store/pg/migrations/020_drop_device_presence_indexes.tx.down.sql @@ -0,0 +1,7 @@ +-- Restores the index set from 001, which reinstates the HOT blocker on the heartbeat and so +-- also restores its write amplification. +CREATE INDEX IF NOT EXISTS devices_last_seen ON devices USING btree (last_seen); + +--bun:split + +CREATE INDEX IF NOT EXISTS devices_disconnected_at ON devices USING btree (disconnected_at); diff --git a/server/api/store/pg/migrations/020_drop_device_presence_indexes.tx.up.sql b/server/api/store/pg/migrations/020_drop_device_presence_indexes.tx.up.sql new file mode 100644 index 00000000000..9d809dde8a8 --- /dev/null +++ b/server/api/store/pg/migrations/020_drop_device_presence_indexes.tx.up.sql @@ -0,0 +1,20 @@ +-- PostgreSQL disqualifies HOT whenever an indexed column changes, and every device presence +-- heartbeat writes last_seen — so this index is what forced each beat to rewrite the heap +-- tuple and touch every index on the table. The device list's ORDER BY last_seen DESC sorts +-- over a sequential scan instead. +-- +-- lock_timeout because migrations run inline in startup before the listener binds: an +-- ACCESS EXCLUSIVE request queued behind a long snapshot would otherwise stall the boot +-- indefinitely. SET LOCAL is enough here because bun runs a .tx. file in one transaction. +SET LOCAL lock_timeout = '60s'; + +--bun:split + +-- IF EXISTS: operators may already have dropped this by hand, and an error fails the boot. +DROP INDEX IF EXISTS devices_last_seen; + +--bun:split + +-- Unused: nothing filters or orders on disconnected_at alone, only as half of the online +-- predicate, which is too unselective to be worth an index scan. +DROP INDEX IF EXISTS devices_disconnected_at; diff --git a/server/api/store/pg/migrations/021_set_devices_fillfactor.tx.down.sql b/server/api/store/pg/migrations/021_set_devices_fillfactor.tx.down.sql new file mode 100644 index 00000000000..fc967e7bd68 --- /dev/null +++ b/server/api/store/pg/migrations/021_set_devices_fillfactor.tx.down.sql @@ -0,0 +1,3 @@ +-- Back to the default 100. Existing pages keep whatever slack they already have; only pages +-- allocated after this are packed full. +ALTER TABLE devices RESET (fillfactor); diff --git a/server/api/store/pg/migrations/021_set_devices_fillfactor.tx.up.sql b/server/api/store/pg/migrations/021_set_devices_fillfactor.tx.up.sql new file mode 100644 index 00000000000..a18bb014063 --- /dev/null +++ b/server/api/store/pg/migrations/021_set_devices_fillfactor.tx.up.sql @@ -0,0 +1,20 @@ +-- HOT needs free space in the same page to write the new tuple version into. Dropping the +-- index in 020 makes a heartbeat HOT-eligible, but eligibility is not enough: at heartbeat +-- rates a densely packed heap runs out of in-page room, falls back to a non-HOT update, and +-- grows until it has accumulated the slack it needed all along. Reserving the slack up front +-- is both cheaper and smaller than letting the table find it by bloating. +-- +-- Measured locally, updating every row of a 58,670-row table six times over, starting from a +-- freshly compacted heap: at the default fillfactor the HOT ratio reached 40.4% and the heap +-- grew 4.7x, against 85.1% and 3.0x at fillfactor 85. Higher HOT means less expansion, so the +-- reserved 15% pays for itself — the fillfactor 85 heap ends up smaller in absolute terms. +-- +-- Must run before 022: VACUUM FULL honours fillfactor when it rewrites the heap (the same +-- 58,670 rows rebuild into 2,257 pages at 100 and 2,667 at 85), so setting this afterwards +-- would leave the compacted pages full and only apply to pages allocated later. +-- +-- The autovacuum_vacuum_scale_factor / autovacuum_analyze_scale_factor knobs proposed +-- alongside this one are deliberately left alone: the same measurement showed no material +-- effect from them once fillfactor is set. They belong with the rest of the server-level +-- tuning rather than here. +ALTER TABLE devices SET (fillfactor = 85); diff --git a/server/api/store/pg/migrations/022_vacuum_full_devices.down.sql b/server/api/store/pg/migrations/022_vacuum_full_devices.down.sql new file mode 100644 index 00000000000..233ce6394b1 --- /dev/null +++ b/server/api/store/pg/migrations/022_vacuum_full_devices.down.sql @@ -0,0 +1,4 @@ +-- This migration rewrites the devices heap and its indexes; it adds no schema and changes no +-- data, so there is nothing to reverse — reintroducing bloat is not something a rollback +-- should do. Leaving it as a no-op keeps the down runnable. +SELECT 1; diff --git a/server/api/store/pg/migrations/022_vacuum_full_devices.up.sql b/server/api/store/pg/migrations/022_vacuum_full_devices.up.sql new file mode 100644 index 00000000000..2212c52095e --- /dev/null +++ b/server/api/store/pg/migrations/022_vacuum_full_devices.up.sql @@ -0,0 +1,34 @@ +-- Reclaims the bloat 020 stops accumulating: while HOT was impossible every heartbeat left a +-- dead tuple behind, and the device list's default sort now scans that heap sequentially. +-- Running after 020 also avoids rebuilding the two indexes that just went away, and running +-- after 021 means the rewrite lays the pages out at that fillfactor. +-- +-- VACUUM cannot run inside a transaction, which is why this file omits the .tx. suffix and +-- keeps every statement in its own --bun:split chunk (see TestNonTransactionalMigrations). +-- +-- The timeouts bound a boot, because migrations run inline before the listener binds. bun marks +-- a migration applied before running it, so if one fires the boot fails but the restart skips +-- this migration and comes up with the table merely still bloated — re-run the VACUUM by hand +-- to finish the job. Peer replicas that boot while this holds the migration lock fail outright +-- rather than wait: migrator.Lock inserts a row, it does not block. +SET lock_timeout = '60s'; + +--bun:split + +SET statement_timeout = '10min'; + +--bun:split + +-- ANALYZE because VACUUM FULL does not refresh planner statistics, and 020 just changed which +-- plans are available for this table. +VACUUM (FULL, ANALYZE) devices; + +--bun:split + +-- SET rather than SET LOCAL above, since there is no transaction to scope it to. The connection +-- returns to the pool unreset, so the timeouts leak into application queries unless undone. +RESET lock_timeout; + +--bun:split + +RESET statement_timeout;