From 28acbcaa8da954f35429577f8fe27ef84ac100c8 Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Tue, 23 Jun 2026 15:36:05 +0600 Subject: [PATCH 1/4] Populate /role_scripts/standby for remote replica coordinator support Signed-off-by: souravbiswassanto --- init_scripts/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/init_scripts/run.sh b/init_scripts/run.sh index 20f9e1d..503d541 100755 --- a/init_scripts/run.sh +++ b/init_scripts/run.sh @@ -11,8 +11,9 @@ export remote_replica=${REMOTE_REPLICA:-false} export source_ssl=${SOURCE_SSL:-false} if [[ $remote_replica == "true" ]]; then - mkdir -p /run_scripts/role + mkdir -p /run_scripts/role /role_scripts/standby cp -r /tmp/role_scripts/$MAJOR_PG_VERSION/standby/* /run_scripts/role/ + cp -r /tmp/role_scripts/$MAJOR_PG_VERSION/standby/* /role_scripts/standby/ elif [[ $STANDALONE == "true" ]]; then mkdir -p /run_scripts/role cp -r /tmp/role_scripts/$MAJOR_PG_VERSION/primary/* /run_scripts/role/ From 084b35fc2066e90c4eb7639958df0b74f3cf02d8 Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Thu, 25 Jun 2026 15:02:43 +0600 Subject: [PATCH 2/4] primary/start.sh: promote from recovery before writes to fork a new timeline When a former standby is started via the primary role script (most importantly the remote-replica -> standalone-HA promotion), standby.signal is present and pg_ctl start brings postgres up in recovery. Previously start.sh removed standby.signal and then ran CREATE DATABASE / ALTER USER writes before the trailing pg_ctl promote; the writes fail under read-only recovery, so on the loop's retry postgres started directly as a primary on the EXISTING timeline and the trailing promote was a no-op. The new HA primary thus stayed on the same timeline as its old source cluster, which on failback forces a full pg_basebackup instead of pg_rewind (a whole-day op at multi-TB scale). Fix: as soon as postgres has started, if standby.signal is present, run pg_ctl promote (which ends recovery, increments the timeline, and clears standby.signal) and wait until pg_is_in_recovery() is false, BEFORE any write. The writes then run against the promoted primary on the new timeline. Scope: only affects a node started via the primary script with standby.signal present (the promotion case). A normal primary start (no standby.signal) and the live-standby fast-failover path (promoted by the coordinator via gRPC, not start.sh) are unchanged. Signed-off-by: souravbiswassanto --- role_scripts/17/primary/start.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/role_scripts/17/primary/start.sh b/role_scripts/17/primary/start.sh index fc05df8..e1e25d3 100755 --- a/role_scripts/17/primary/start.sh +++ b/role_scripts/17/primary/start.sh @@ -95,6 +95,25 @@ export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} psql=(psql -v ON_ERROR_STOP=1) +# If standby.signal is present, postgres started in recovery above (this node was a standby being +# promoted). Promote it NOW to fork a NEW timeline BEFORE any write below. This is essential for the +# remote-replica -> standalone-HA transition: without promoting from recovery the node comes up on +# the SAME timeline as its former source cluster, which on failback forces a full pg_basebackup +# instead of pg_rewind. Promotion ends recovery, increments the timeline, and removes standby.signal. +# A normal primary start (no standby.signal) skips this block, so fresh bootstrap and the live-standby +# fast-failover path (which promotes via the coordinator, not start.sh) are unaffected. +if [[ -f "/var/pv/data/standby.signal" ]]; then + echo "standby.signal present -> promoting to fork a new timeline before writes" + pg_ctl -D "$PGDATA" promote || true + for _ in $(seq 1 120); do + if [[ "$("${psql[@]}" --username postgres -tAc "SELECT pg_is_in_recovery();" 2>/dev/null)" == "f" ]]; then + echo "promotion complete; node is now primary on a new timeline" + break + fi + sleep 1 + done +fi + # create database with specified name if [ "$POSTGRES_DB" != "postgres" ]; then "${psql[@]}" --username postgres <<-EOSQL From 3b816f86a3712f62416851bd48d176216d4010d0 Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Thu, 25 Jun 2026 15:17:56 +0600 Subject: [PATCH 3/4] primary/start.sh: use pg_controldata (not a DB connection) to await promotion The post-promote wait must not depend on connecting as the postgres superuser: if that role is missing/renamed the psql probe fails and the loop burns its full 120s timeout. Poll pg_controldata's cluster state (in archive recovery -> in production) instead, which needs no DB connection or role. Signed-off-by: souravbiswassanto --- role_scripts/17/primary/start.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/role_scripts/17/primary/start.sh b/role_scripts/17/primary/start.sh index e1e25d3..d636bb6 100755 --- a/role_scripts/17/primary/start.sh +++ b/role_scripts/17/primary/start.sh @@ -105,8 +105,12 @@ psql=(psql -v ON_ERROR_STOP=1) if [[ -f "/var/pv/data/standby.signal" ]]; then echo "standby.signal present -> promoting to fork a new timeline before writes" pg_ctl -D "$PGDATA" promote || true + # Wait until recovery has ended using pg_controldata (no DB connection / role dependency, so a + # missing or renamed superuser role can never stall this loop). State goes from + # "in archive recovery" to "in production" once promotion completes. for _ in $(seq 1 120); do - if [[ "$("${psql[@]}" --username postgres -tAc "SELECT pg_is_in_recovery();" 2>/dev/null)" == "f" ]]; then + state=$(pg_controldata "$PGDATA" 2>/dev/null | grep "Database cluster state" | sed 's/.*:[[:space:]]*//') + if [[ "$state" == "in production" ]]; then echo "promotion complete; node is now primary on a new timeline" break fi From 33df0a80c41ae9606d36affd53c772c642616d3f Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Thu, 25 Jun 2026 20:52:17 +0600 Subject: [PATCH 4/4] primary/start.sh: port promote-from-recovery to PG 13-16,18 Apply the same promote-before-writes fix already in PG17 to the other supported major versions (13,14,15,16,18) so the remote-replica -> standalone-HA timeline bump works regardless of PostgreSQL version. When standby.signal is present at startup (a former standby being promoted), pg_ctl promote runs before the CREATE DATABASE / ALTER USER writes, forking a new timeline so failback uses pg_rewind instead of a full pg_basebackup. Versions < 13 are out of scope. Signed-off-by: souravbiswassanto --- role_scripts/13/primary/start.sh | 20 ++++++++++++++++++++ role_scripts/14/primary/start.sh | 20 ++++++++++++++++++++ role_scripts/15/primary/start.sh | 20 ++++++++++++++++++++ role_scripts/16/primary/start.sh | 20 ++++++++++++++++++++ role_scripts/18/primary/start.sh | 20 ++++++++++++++++++++ 5 files changed, 100 insertions(+) diff --git a/role_scripts/13/primary/start.sh b/role_scripts/13/primary/start.sh index c4ab448..0675b8d 100755 --- a/role_scripts/13/primary/start.sh +++ b/role_scripts/13/primary/start.sh @@ -91,6 +91,26 @@ export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} psql=(psql -v ON_ERROR_STOP=1) +# If standby.signal is present, postgres started in recovery above (this node was a standby being +# promoted). Promote it NOW to fork a NEW timeline BEFORE any write below. This is essential for the +# remote-replica -> standalone-HA transition: without promoting from recovery the node comes up on +# the SAME timeline as its former source cluster, which on failback forces a full pg_basebackup +# instead of pg_rewind. Promotion ends recovery, increments the timeline, and removes standby.signal. +# A normal primary start (no standby.signal) skips this block, so fresh bootstrap and the live-standby +# fast-failover path (which promotes via the coordinator, not start.sh) are unaffected. +if [[ -f "/var/pv/data/standby.signal" ]]; then + echo "standby.signal present -> promoting to fork a new timeline before writes" + pg_ctl -D "$PGDATA" promote || true + for _ in $(seq 1 120); do + state=$(pg_controldata "$PGDATA" 2>/dev/null | grep "Database cluster state" | sed 's/.*:[[:space:]]*//') + if [[ "$state" == "in production" ]]; then + echo "promotion complete; node is now primary on a new timeline" + break + fi + sleep 1 + done +fi + # create database with specified name if [ "$POSTGRES_DB" != "postgres" ]; then "${psql[@]}" --username postgres <<-EOSQL diff --git a/role_scripts/14/primary/start.sh b/role_scripts/14/primary/start.sh index cf42d90..2202449 100755 --- a/role_scripts/14/primary/start.sh +++ b/role_scripts/14/primary/start.sh @@ -97,6 +97,26 @@ export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} psql=(psql -v ON_ERROR_STOP=1) +# If standby.signal is present, postgres started in recovery above (this node was a standby being +# promoted). Promote it NOW to fork a NEW timeline BEFORE any write below. This is essential for the +# remote-replica -> standalone-HA transition: without promoting from recovery the node comes up on +# the SAME timeline as its former source cluster, which on failback forces a full pg_basebackup +# instead of pg_rewind. Promotion ends recovery, increments the timeline, and removes standby.signal. +# A normal primary start (no standby.signal) skips this block, so fresh bootstrap and the live-standby +# fast-failover path (which promotes via the coordinator, not start.sh) are unaffected. +if [[ -f "/var/pv/data/standby.signal" ]]; then + echo "standby.signal present -> promoting to fork a new timeline before writes" + pg_ctl -D "$PGDATA" promote || true + for _ in $(seq 1 120); do + state=$(pg_controldata "$PGDATA" 2>/dev/null | grep "Database cluster state" | sed 's/.*:[[:space:]]*//') + if [[ "$state" == "in production" ]]; then + echo "promotion complete; node is now primary on a new timeline" + break + fi + sleep 1 + done +fi + # create database with specified name if [ "$POSTGRES_DB" != "postgres" ]; then "${psql[@]}" --username postgres <<-EOSQL diff --git a/role_scripts/15/primary/start.sh b/role_scripts/15/primary/start.sh index c628a37..6bc863b 100755 --- a/role_scripts/15/primary/start.sh +++ b/role_scripts/15/primary/start.sh @@ -95,6 +95,26 @@ export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} psql=(psql -v ON_ERROR_STOP=1) +# If standby.signal is present, postgres started in recovery above (this node was a standby being +# promoted). Promote it NOW to fork a NEW timeline BEFORE any write below. This is essential for the +# remote-replica -> standalone-HA transition: without promoting from recovery the node comes up on +# the SAME timeline as its former source cluster, which on failback forces a full pg_basebackup +# instead of pg_rewind. Promotion ends recovery, increments the timeline, and removes standby.signal. +# A normal primary start (no standby.signal) skips this block, so fresh bootstrap and the live-standby +# fast-failover path (which promotes via the coordinator, not start.sh) are unaffected. +if [[ -f "/var/pv/data/standby.signal" ]]; then + echo "standby.signal present -> promoting to fork a new timeline before writes" + pg_ctl -D "$PGDATA" promote || true + for _ in $(seq 1 120); do + state=$(pg_controldata "$PGDATA" 2>/dev/null | grep "Database cluster state" | sed 's/.*:[[:space:]]*//') + if [[ "$state" == "in production" ]]; then + echo "promotion complete; node is now primary on a new timeline" + break + fi + sleep 1 + done +fi + # create database with specified name if [ "$POSTGRES_DB" != "postgres" ]; then "${psql[@]}" --username postgres <<-EOSQL diff --git a/role_scripts/16/primary/start.sh b/role_scripts/16/primary/start.sh index 0d1c333..a063073 100755 --- a/role_scripts/16/primary/start.sh +++ b/role_scripts/16/primary/start.sh @@ -94,6 +94,26 @@ export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} psql=(psql -v ON_ERROR_STOP=1) +# If standby.signal is present, postgres started in recovery above (this node was a standby being +# promoted). Promote it NOW to fork a NEW timeline BEFORE any write below. This is essential for the +# remote-replica -> standalone-HA transition: without promoting from recovery the node comes up on +# the SAME timeline as its former source cluster, which on failback forces a full pg_basebackup +# instead of pg_rewind. Promotion ends recovery, increments the timeline, and removes standby.signal. +# A normal primary start (no standby.signal) skips this block, so fresh bootstrap and the live-standby +# fast-failover path (which promotes via the coordinator, not start.sh) are unaffected. +if [[ -f "/var/pv/data/standby.signal" ]]; then + echo "standby.signal present -> promoting to fork a new timeline before writes" + pg_ctl -D "$PGDATA" promote || true + for _ in $(seq 1 120); do + state=$(pg_controldata "$PGDATA" 2>/dev/null | grep "Database cluster state" | sed 's/.*:[[:space:]]*//') + if [[ "$state" == "in production" ]]; then + echo "promotion complete; node is now primary on a new timeline" + break + fi + sleep 1 + done +fi + # create database with specified name if [ "$POSTGRES_DB" != "postgres" ]; then "${psql[@]}" --username postgres <<-EOSQL diff --git a/role_scripts/18/primary/start.sh b/role_scripts/18/primary/start.sh index c3ba065..7610a4a 100755 --- a/role_scripts/18/primary/start.sh +++ b/role_scripts/18/primary/start.sh @@ -95,6 +95,26 @@ export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} psql=(psql -v ON_ERROR_STOP=1) +# If standby.signal is present, postgres started in recovery above (this node was a standby being +# promoted). Promote it NOW to fork a NEW timeline BEFORE any write below. This is essential for the +# remote-replica -> standalone-HA transition: without promoting from recovery the node comes up on +# the SAME timeline as its former source cluster, which on failback forces a full pg_basebackup +# instead of pg_rewind. Promotion ends recovery, increments the timeline, and removes standby.signal. +# A normal primary start (no standby.signal) skips this block, so fresh bootstrap and the live-standby +# fast-failover path (which promotes via the coordinator, not start.sh) are unaffected. +if [[ -f "/var/pv/data/standby.signal" ]]; then + echo "standby.signal present -> promoting to fork a new timeline before writes" + pg_ctl -D "$PGDATA" promote || true + for _ in $(seq 1 120); do + state=$(pg_controldata "$PGDATA" 2>/dev/null | grep "Database cluster state" | sed 's/.*:[[:space:]]*//') + if [[ "$state" == "in production" ]]; then + echo "promotion complete; node is now primary on a new timeline" + break + fi + sleep 1 + done +fi + # create database with specified name if [ "$POSTGRES_DB" != "postgres" ]; then "${psql[@]}" --username postgres <<-EOSQL