perf(server): stop the device heartbeat from blocking HOT updates

server/api/store/pg/migrations/020_drop_device_presence_indexes.tx.down.sql

@@ -0,0 +1,7 @@
+-- Restores the index set from 001, which reinstates the HOT blocker on the heartbeat and so
+-- also restores its write amplification.
+CREATE INDEX IF NOT EXISTS devices_last_seen ON devices USING btree (last_seen);
+
+--bun:split
+
+CREATE INDEX IF NOT EXISTS devices_disconnected_at ON devices USING btree (disconnected_at);

server/api/store/pg/migrations/020_drop_device_presence_indexes.tx.up.sql

@@ -0,0 +1,20 @@
+-- PostgreSQL disqualifies HOT whenever an indexed column changes, and every device presence
+-- heartbeat writes last_seen — so this index is what forced each beat to rewrite the heap
+-- tuple and touch every index on the table. The device list's ORDER BY last_seen DESC sorts
+-- over a sequential scan instead.
+--
+-- lock_timeout because migrations run inline in startup before the listener binds: an
+-- ACCESS EXCLUSIVE request queued behind a long snapshot would otherwise stall the boot
+-- indefinitely. SET LOCAL is enough here because bun runs a .tx. file in one transaction.
+SET LOCAL lock_timeout = '60s';
+
+--bun:split
+
+-- IF EXISTS: operators may already have dropped this by hand, and an error fails the boot.
+DROP INDEX IF EXISTS devices_last_seen;
+
+--bun:split
+
+-- Unused: nothing filters or orders on disconnected_at alone, only as half of the online
+-- predicate, which is too unselective to be worth an index scan.
+DROP INDEX IF EXISTS devices_disconnected_at;

server/api/store/pg/migrations/021_set_devices_fillfactor.tx.down.sql

@@ -0,0 +1,3 @@
+-- Back to the default 100. Existing pages keep whatever slack they already have; only pages
+-- allocated after this are packed full.
+ALTER TABLE devices RESET (fillfactor);

server/api/store/pg/migrations/021_set_devices_fillfactor.tx.up.sql

@@ -0,0 +1,20 @@
+-- HOT needs free space in the same page to write the new tuple version into. Dropping the
+-- index in 020 makes a heartbeat HOT-eligible, but eligibility is not enough: at heartbeat
+-- rates a densely packed heap runs out of in-page room, falls back to a non-HOT update, and
+-- grows until it has accumulated the slack it needed all along. Reserving the slack up front
+-- is both cheaper and smaller than letting the table find it by bloating.
+--
+-- Measured locally, updating every row of a 58,670-row table six times over, starting from a
+-- freshly compacted heap: at the default fillfactor the HOT ratio reached 40.4% and the heap
+-- grew 4.7x, against 85.1% and 3.0x at fillfactor 85. Higher HOT means less expansion, so the
+-- reserved 15% pays for itself — the fillfactor 85 heap ends up smaller in absolute terms.
+--
+-- Must run before 022: VACUUM FULL honours fillfactor when it rewrites the heap (the same
+-- 58,670 rows rebuild into 2,257 pages at 100 and 2,667 at 85), so setting this afterwards
+-- would leave the compacted pages full and only apply to pages allocated later.
+--
+-- The autovacuum_vacuum_scale_factor / autovacuum_analyze_scale_factor knobs proposed
+-- alongside this one are deliberately left alone: the same measurement showed no material
+-- effect from them once fillfactor is set. They belong with the rest of the server-level
+-- tuning rather than here.
+ALTER TABLE devices SET (fillfactor = 85);

server/api/store/pg/migrations/022_vacuum_full_devices.down.sql

@@ -0,0 +1,4 @@
+-- This migration rewrites the devices heap and its indexes; it adds no schema and changes no
+-- data, so there is nothing to reverse — reintroducing bloat is not something a rollback
+-- should do. Leaving it as a no-op keeps the down runnable.
+SELECT 1;

server/api/store/pg/migrations/022_vacuum_full_devices.up.sql

@@ -0,0 +1,34 @@
+-- Reclaims the bloat 020 stops accumulating: while HOT was impossible every heartbeat left a
+-- dead tuple behind, and the device list's default sort now scans that heap sequentially.
+-- Running after 020 also avoids rebuilding the two indexes that just went away, and running
+-- after 021 means the rewrite lays the pages out at that fillfactor.
+--
+-- VACUUM cannot run inside a transaction, which is why this file omits the .tx. suffix and
+-- keeps every statement in its own --bun:split chunk (see TestNonTransactionalMigrations).
+--
+-- The timeouts bound a boot, because migrations run inline before the listener binds. bun marks
+-- a migration applied before running it, so if one fires the boot fails but the restart skips
+-- this migration and comes up with the table merely still bloated — re-run the VACUUM by hand
+-- to finish the job. Peer replicas that boot while this holds the migration lock fail outright
+-- rather than wait: migrator.Lock inserts a row, it does not block.
+SET lock_timeout = '60s';
+
+--bun:split
+
+SET statement_timeout = '10min';
+
+--bun:split
+
+-- ANALYZE because VACUUM FULL does not refresh planner statistics, and 020 just changed which
+-- plans are available for this table.
+VACUUM (FULL, ANALYZE) devices;
+
+--bun:split
+
+-- SET rather than SET LOCAL above, since there is no transaction to scope it to. The connection
+-- returns to the pool unreset, so the timeouts leak into application queries unless undone.
+RESET lock_timeout;
+
+--bun:split
+
+RESET statement_timeout;