Upgrade encore image to PSQL 18

.github/workflows/cd.yml

@@ -14,22 +14,22 @@ on:
 jobs:
   publish-docker-images:
     name: "publish docker images"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to Docker Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -38,7 +38,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: encoredotdev/postgres
           labels: |
@@ -54,7 +54,7 @@ jobs:
             type=ref,event=pr
 
       - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64

Dockerfile

@@ -1,6 +1,6 @@
 # This file is inspired by github.com/neondatabase/neon's docker file.
 
-ARG PG_MAJOR=15
+ARG PG_MAJOR=18
 ARG EXTENSION_DIR=/usr/share/postgresql/${PG_MAJOR}/extension
 ARG INCLUDE_DIR=/usr/include/postgresql/${PG_MAJOR}
 ARG LIB_DIR=/usr/lib/postgresql/${PG_MAJOR}
@@ -14,10 +14,10 @@ ARG PGCONFIG=${BIN_DIR}/pg_config
 #
 #########################################################################################
 
-FROM postgres:${PG_MAJOR}-bullseye AS pg-build
+FROM postgres:${PG_MAJOR}-trixie AS pg-build
 
 RUN apt update && \
-    apt install -y postgresql-server-dev-$PG_MAJOR 
+    apt install -y postgresql-server-dev-$PG_MAJOR
 
 
 #########################################################################################
@@ -26,7 +26,7 @@ RUN apt update && \
 # Used to copy postgres header files from, without needing postgres installed
 #
 #########################################################################################
-FROM debian:bullseye-slim AS build-deps
+FROM debian:trixie-slim AS build-deps
 
 ARG EXTENSION_DIR
 ARG LIB_DIR
@@ -40,7 +40,7 @@ ENV CXX=g++
 RUN apt update && \
     apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \
     zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev \
-    libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd clang-13
+    libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd clang-19
 
 
 #########################################################################################
@@ -56,8 +56,8 @@ ARG LIB_DIR
 ARG INCLUDE_DIR
 ARG PGCONFIG
 
-ENV PGVECTOR_VERSION 0.7.0
-ENV PGVECTOR_SHA 1b5503a35c265408b6eb282621c5e1e75f7801afc04eecb950796cfee2e3d1d8
+ENV PGVECTOR_VERSION=0.8.2
+ENV PGVECTOR_SHA=69f4019389af05dc1c9548deb8628e62878e6e207c03907f2b8af2016472cdaa
 
 COPY --from=pg-build ${EXTENSION_DIR}/ ${EXTENSION_DIR}/
 COPY --from=pg-build ${LIB_DIR}/ ${LIB_DIR}/
@@ -74,20 +74,20 @@ RUN mkdir /out /out/lib /out/share /out/share/extension && \
     cp ${EXTENSION_DIR}/vector* /out/share/extension/ && \
     echo 'trusted = true' >> /out/share/extension/vector.control
 
-    
+
 #########################################################################################
 #
 # Final image
 #
 #########################################################################################
 
-FROM postgres:${PG_MAJOR}-bullseye
+FROM postgres:${PG_MAJOR}-trixie
 
 LABEL maintainer="Encore - https://encore.dev"
 ARG EXTENSION_DIR
 ARG LIB_DIR
 
-ENV POSTGIS_MAJOR 3
+ENV POSTGIS_MAJOR=3
 
 RUN apt update \
     && apt install -y --no-install-recommends \
@@ -114,3 +114,30 @@ RUN for ext in address_standardizer address_standardizer-3 address_standardizer_
             echo "trusted = true" >> "$EXTENSION_DIR/$ext.control"; \
         fi \
     done
+
+# Raise max_connections on first init. initdb's -c writes the setting into
+# the generated postgresql.conf; editing postgresql.conf.sample alone is
+# not enough because initdb recomputes max_connections from kernel limits.
+ENV POSTGRES_INITDB_ARGS="-c max_connections=1000"
+
+# Install legacy PostgreSQL server binaries plus matching extension
+# libraries (postgis, pgvector) so the OLD cluster can load extensions
+# during pg_upgrade's schema dump (opt-in via AUTO_PG_UPGRADE=1). Override
+# at build time with --build-arg LEGACY_PG_VERSIONS="15 16 17" to support more.
+ARG LEGACY_PG_VERSIONS="15"
+RUN mkdir -p /etc/postgresql-common && \
+    echo 'create_main_cluster = false' > /etc/postgresql-common/createcluster.conf && \
+    apt update && \
+    for v in $LEGACY_PG_VERSIONS; do \
+        apt install -y --no-install-recommends \
+            postgresql-$v \
+            postgresql-$v-postgis-$POSTGIS_MAJOR \
+            postgresql-$v-pgvector; \
+    done && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY scripts/auto-upgrade-entrypoint.sh scripts/pg-auto-upgrade.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/auto-upgrade-entrypoint.sh /usr/local/bin/pg-auto-upgrade.sh
+
+ENTRYPOINT ["auto-upgrade-entrypoint.sh"]
+CMD ["postgres"]

scripts/auto-upgrade-entrypoint.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -e
+
+# Opt-in: only run the auto-upgrade probe when AUTO_PG_UPGRADE=1.
+# When enabled, detects a PG_VERSION mismatch between the mounted PGDATA
+# and this image, and runs pg_upgrade as the postgres user before chaining
+# to the official entrypoint.
+if [ "${AUTO_PG_UPGRADE:-0}" = "1" ] \
+        && [ "$(id -u)" = "0" ] \
+        && [ -s "${PGDATA}/PG_VERSION" ]; then
+    OLD_VER="$(cat "${PGDATA}/PG_VERSION")"
+    if [ "$OLD_VER" != "$PG_MAJOR" ]; then
+        chown -R postgres:postgres "$PGDATA" "$(dirname "$PGDATA")"
+        gosu postgres /usr/local/bin/pg-auto-upgrade.sh
+    fi
+fi
+
+exec docker-entrypoint.sh "$@"

scripts/pg-auto-upgrade.sh

@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+OLD_VER="$(cat "${PGDATA}/PG_VERSION")"
+NEW_VER="${PG_MAJOR}"
+
+if [ "$OLD_VER" = "$NEW_VER" ]; then
+    exit 0
+fi
+
+if [ "$OLD_VER" -gt "$NEW_VER" ]; then
+    echo "pg-auto-upgrade: refusing to downgrade PGDATA from PG${OLD_VER} to PG${NEW_VER}" >&2
+    exit 1
+fi
+
+OLD_BIN="/usr/lib/postgresql/${OLD_VER}/bin"
+NEW_BIN="/usr/lib/postgresql/${NEW_VER}/bin"
+
+if [ ! -x "$OLD_BIN/pg_upgrade" ]; then
+    echo "pg-auto-upgrade: PG${OLD_VER} binaries are not installed in this image; cannot upgrade." >&2
+    echo "pg-auto-upgrade: rebuild with LEGACY_PG_VERSIONS including ${OLD_VER}." >&2
+    exit 1
+fi
+
+echo "pg-auto-upgrade: migrating PGDATA from PG${OLD_VER} to PG${NEW_VER}"
+
+# Stage both clusters as subdirectories of PGDATA itself. This keeps them
+# on the same filesystem (required for pg_upgrade --link) regardless of
+# whether the user mounted the volume at PGDATA or its parent.
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+SUBDIR_OLD="${PGDATA}/.upgrade-old-${TS}"
+SUBDIR_NEW="${PGDATA}/.upgrade-new-${TS}"
+
+mkdir -p "$SUBDIR_OLD" "$SUBDIR_NEW"
+chmod 700 "$SUBDIR_OLD" "$SUBDIR_NEW"
+
+# Roll back only failures that happen before pg_upgrade --link has run.
+ROLLBACK_SAFE=1
+on_failure() {
+    if [ "$ROLLBACK_SAFE" != "1" ]; then
+        return
+    fi
+    echo "pg-auto-upgrade: failed before pg_upgrade ran; restoring PGDATA contents" >&2
+    rm -rf "$SUBDIR_NEW" 2>/dev/null || true
+    if [ -d "$SUBDIR_OLD" ]; then
+        find "$SUBDIR_OLD" -mindepth 1 -maxdepth 1 \
+            -exec mv -t "$PGDATA" {} + 2>/dev/null || true
+        rmdir "$SUBDIR_OLD" 2>/dev/null || true
+    fi
+}
+trap on_failure ERR
+
+# Move existing cluster contents into SUBDIR_OLD (skip the upgrade subdirs themselves).
+find "$PGDATA" -mindepth 1 -maxdepth 1 \
+    ! -path "$SUBDIR_OLD" ! -path "$SUBDIR_NEW" \
+    -exec mv -t "$SUBDIR_OLD" {} +
+
+# Match the new cluster's data-checksum setting to the old cluster's;
+# pg_upgrade refuses if they differ.
+OLD_CHECKSUM_VER="$("$OLD_BIN/pg_controldata" "$SUBDIR_OLD" \
+    | awk -F': *' '/Data page checksum version/{print $2}')"
+if [ "${OLD_CHECKSUM_VER:-0}" -gt 0 ]; then
+    CHECKSUM_ARG="--data-checksums"
+else
+    CHECKSUM_ARG="--no-data-checksums"
+fi
+
+"$NEW_BIN/initdb" \
+    --username="${POSTGRES_USER:-postgres}" \
+    "$CHECKSUM_ARG" \
+    -c max_connections=1000 \
+    -D "$SUBDIR_NEW"
+
+cd /tmp
+ROLLBACK_SAFE=0
+"$NEW_BIN/pg_upgrade" \
+    --old-bindir="$OLD_BIN" \
+    --new-bindir="$NEW_BIN" \
+    --old-datadir="$SUBDIR_OLD" \
+    --new-datadir="$SUBDIR_NEW" \
+    --link
+
+# Promote the new cluster's files into PGDATA up front, so that even if
+# a later step fails the volume contains a startable cluster.
+find "$SUBDIR_NEW" -mindepth 1 -maxdepth 1 -exec mv -t "$PGDATA" {} +
+rmdir "$SUBDIR_NEW"
+
+# Refresh collation versions: the OS glibc may differ from when the old
+# cluster was created (e.g. bullseye -> trixie), which silently breaks
+# b-tree ordering. REINDEX rebuilds indexes against current collation
+# rules; REFRESH updates the catalog's recorded version.
+refresh_collations() {
+    local pg_temp_port=50432
+    local pg_temp_log=/tmp/pg_post_upgrade.log
+
+    "$NEW_BIN/pg_ctl" -D "$PGDATA" \
+        -o "-c listen_addresses='' -c unix_socket_directories=/tmp -p $pg_temp_port" \
+        -l "$pg_temp_log" -w start
+
+    local psql=("$NEW_BIN/psql" -h /tmp -p "$pg_temp_port" \
+        -U "${POSTGRES_USER:-postgres}" -X -At -v ON_ERROR_STOP=1)
+
+    local dbs
+    dbs="$("${psql[@]}" -d postgres -c "
+        SELECT datname FROM pg_database
+        WHERE datallowconn AND datname <> 'template0'
+          AND datcollversion IS DISTINCT FROM pg_database_collation_actual_version(oid)
+    ")" || dbs=""
+
+    local db
+    while read -r db; do
+        [ -n "$db" ] || continue
+        echo "pg-auto-upgrade: REINDEX + REFRESH for database $db"
+        "${psql[@]}" -d "$db" -c "REINDEX DATABASE \"$db\""
+        "${psql[@]}" -d "$db" -c "ALTER DATABASE \"$db\" REFRESH COLLATION VERSION"
+        "${psql[@]}" -d "$db" <<'SQL'
+DO $$
+DECLARE r record;
+BEGIN
+  FOR r IN SELECT n.nspname, c.collname
+           FROM pg_collation c JOIN pg_namespace n ON c.collnamespace = n.oid
+           WHERE c.collversion IS NOT NULL
+             AND c.collversion <> pg_collation_actual_version(c.oid)
+  LOOP
+    EXECUTE format('ALTER COLLATION %I.%I REFRESH VERSION', r.nspname, r.collname);
+  END LOOP;
+END
+$$;
+SQL
+    done <<< "$dbs"
+
+    "$NEW_BIN/pg_ctl" -D "$PGDATA" -w stop
+}
+
+if ! refresh_collations; then
+    echo "pg-auto-upgrade: collation refresh failed; the cluster is upgraded but" >&2
+    echo "pg-auto-upgrade: you should manually REINDEX and REFRESH affected databases." >&2
+    "$NEW_BIN/pg_ctl" -D "$PGDATA" -w stop 2>/dev/null || true
+fi
+
+# Carry user-defined config and ALTER SYSTEM settings forward.
+for f in postgresql.conf postgresql.auto.conf pg_hba.conf pg_ident.conf; do
+    if [ -f "$SUBDIR_OLD/$f" ]; then
+        cp "$SUBDIR_OLD/$f" "$PGDATA/$f"
+    fi
+done
+
+# Re-apply image defaults that should survive an upgrade.
+sed -ri "s/^#?max_connections\s*=.*/max_connections = 1000/" "$PGDATA/postgresql.conf"
+
+echo "pg-auto-upgrade: PG${OLD_VER} -> PG${NEW_VER} complete"
+echo "pg-auto-upgrade: pre-upgrade files kept at $SUBDIR_OLD — delete after verifying the new cluster"