From 36fe7c6cd9ddfe603d12f42c38faad9dc0bb9c83 Mon Sep 17 00:00:00 2001 From: Pedro Yves Fracari Date: Thu, 6 Aug 2026 10:05:33 -0300 Subject: [PATCH 1/7] refactor(api): extract the Prometheus gauge parser for reuse The /metrics scrape and its gauge parser were private to the sync-progress handler. The readiness probe needs the same two pieces, so move them into src/api/prometheus.ts unchanged. --- src/api/endpoints/sync-progress.ts | 30 ++---------------------------- src/api/prometheus.ts | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 28 deletions(-) create mode 100644 src/api/prometheus.ts diff --git a/src/api/endpoints/sync-progress.ts b/src/api/endpoints/sync-progress.ts index a856b12..b38ac69 100644 --- a/src/api/endpoints/sync-progress.ts +++ b/src/api/endpoints/sync-progress.ts @@ -1,37 +1,11 @@ import type { RouteHandler } from "@hono/zod-openapi"; import type { syncProgressRoute } from "../routes"; - -// Prometheus text-format parser for a single gauge metric. -// Matches lines like: metric_name{label="value"} 123 -const GAUGE_RE = /^(\w+)\{([^}]*)\}\s+([\d.]+)/; - -function parsePrometheusGauge( - lines: string[], - metricName: string, -): Map { - const result = new Map(); - for (const line of lines) { - if (!line.startsWith(metricName + "{")) continue; - const m = GAUGE_RE.exec(line); - if (!m) continue; - const labels = m[2] as string; - const value = Number(m[3]); - // Extract chain label value - const chainMatch = /chain="([^"]+)"/.exec(labels); - if (chainMatch) result.set(chainMatch[1] as string, value); - } - return result; -} +import { fetchMetricLines, parsePrometheusGauge } from "../prometheus"; export const syncProgressHandler: RouteHandler = async (c) => { // Resolve /metrics relative to the current request so this works on any port. - const origin = new URL(c.req.url).origin; - const metricsText = await fetch(`${origin}/metrics`) - .then((r) => r.text()) - .catch(() => ""); - - const lines = metricsText.split("\n"); + const lines = await fetchMetricLines(new URL(c.req.url).origin); const total = parsePrometheusGauge(lines, "ponder_historical_total_blocks"); const completed = parsePrometheusGauge( diff --git a/src/api/prometheus.ts b/src/api/prometheus.ts new file mode 100644 index 0000000..f941a89 --- /dev/null +++ b/src/api/prometheus.ts @@ -0,0 +1,29 @@ +// Prometheus text-format parser for a single gauge metric. +// Matches lines like: metric_name{label="value"} 123 +const GAUGE_RE = /^(\w+)\{([^}]*)\}\s+([\d.]+)/; + +/** Parse one gauge from Ponder's /metrics output into a map keyed by chain label. */ +export function parsePrometheusGauge( + lines: string[], + metricName: string, +): Map { + const result = new Map(); + for (const line of lines) { + if (!line.startsWith(metricName + "{")) continue; + const m = GAUGE_RE.exec(line); + if (!m) continue; + const labels = m[2] as string; + const value = Number(m[3]); + const chainMatch = /chain="([^"]+)"/.exec(labels); + if (chainMatch) result.set(chainMatch[1] as string, value); + } + return result; +} + +/** Fetch Ponder's /metrics on the same origin and split it into lines. */ +export async function fetchMetricLines(origin: string): Promise { + const text = await fetch(`${origin}/metrics`) + .then((r) => r.text()) + .catch(() => ""); + return text.split("\n"); +} From 36793c4c71a6246d24a7cf475936a7159b279e0d Mon Sep 17 00:00:00 2001 From: Pedro Yves Fracari Date: Thu, 6 Aug 2026 10:05:43 -0300 Subject: [PATCH 2/7] feat(api): gate /readyz on per-chain block freshness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ponder's /ready latches at 200 once historical sync finishes and never goes back, so /readyz inherited a blind spot: an indexer whose realtime subscription dies keeps reporting itself ready while its data ages. In production a dRPC WebSocket stopped delivering newHeads without closing the socket; Ponder logged one warning and idled, and every probe stayed green for a week. /readyz now also reads ponder_sync_block_timestamp per active chain and fails when the newest synced block is older than READINESS_MAX_LAG_SECONDS (default 300, per-chain override with a numeric chain-id suffix). The 503 body names the chain, its block number, and the measured lag. Compares against wall-clock rather than calling eth_blockNumber on purpose: a readiness probe that depends on the RPC turns an RPC outage into a restart loop, and the block timestamp already carries enough to spot a stalled sync. The trade-off is that a genuine chain halt reads as staleness. A chain missing from the metrics counts as stale — absent data is not evidence of freshness. --- .env.example | 8 ++ src/api/freshness.ts | 79 ++++++++++++++++++ src/api/index.ts | 33 +++++--- tests/api/freshness.test.ts | 155 ++++++++++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 src/api/freshness.ts create mode 100644 tests/api/freshness.test.ts diff --git a/.env.example b/.env.example index 8812e3f..3129941 100644 --- a/.env.example +++ b/.env.example @@ -60,6 +60,14 @@ DATABASE_SCHEMA=programmatic_orders # ETH_GET_LOGS_BLOCK_RANGE_100=5000 # gnosis +# How stale a chain's newest synced block may get before /readyz reports 503 and the +# container healthcheck starts failing. Seconds; default 300. Raise it if normal +# indexing latency on a chain regularly exceeds the budget; lower it to detect a +# stalled subscription sooner. Per-chain override takes the numeric chain-id suffix. +# READINESS_MAX_LAG_SECONDS=300 +# READINESS_MAX_LAG_SECONDS_1=300 # mainnet +# READINESS_MAX_LAG_SECONDS_100=300 # gnosis + # Logging (optional) # PINO_LOG_LEVEL=info diff --git a/src/api/freshness.ts b/src/api/freshness.ts new file mode 100644 index 0000000..8b09388 --- /dev/null +++ b/src/api/freshness.ts @@ -0,0 +1,79 @@ +import { ACTIVE_CHAINS } from "../chains"; +import type { ChainConfig } from "../chains/types"; + +/** + * How far the newest synced block may fall behind wall-clock before a chain is + * considered stalled. Generous by default: the live block handlers + * (OrderDiscoveryPoller, CandidateConfirmer, …) each add seconds of latency per + * firing, so normal operation already lags tens of seconds behind the tip. + */ +export const DEFAULT_MAX_LAG_SECONDS = 300; + +/** Resolve the staleness budget for a chain: per-chain env, then global env, then default. */ +export function maxLagSecondsFor(chainId: number): number { + for (const raw of [ + process.env[`READINESS_MAX_LAG_SECONDS_${chainId}`], + process.env.READINESS_MAX_LAG_SECONDS, + ]) { + const parsed = Number(raw); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + return DEFAULT_MAX_LAG_SECONDS; +} + +export type StaleChain = { + chain: string; + /** Newest synced block number, or null when the chain reports no block at all. */ + blockNumber: number | null; + /** Seconds between that block's timestamp and now, or null when unknown. */ + lagSeconds: number | null; + maxLagSeconds: number; +}; + +/** + * Compare each active chain's newest synced block against wall-clock time. + * + * Wall-clock rather than a fresh `eth_blockNumber` call on purpose: a readiness + * probe that depends on the RPC turns an RPC outage into a restart loop, and the + * block timestamp already carries everything needed to spot a stalled sync. + * The trade-off is that a genuine chain halt reads as staleness here. + * + * A chain missing from the metrics counts as stale — absent data is not evidence + * of freshness. + */ +export function findStaleChains( + blockTimestamps: Map, + blockNumbers: Map, + nowSeconds: number, + chains: ChainConfig[] = ACTIVE_CHAINS, +): StaleChain[] { + const stale: StaleChain[] = []; + + for (const chain of chains) { + const maxLagSeconds = maxLagSecondsFor(chain.chainId); + const blockNumber = blockNumbers.get(chain.name) ?? null; + const timestamp = blockTimestamps.get(chain.name); + + if (timestamp === undefined || timestamp <= 0) { + stale.push({ chain: chain.name, blockNumber, lagSeconds: null, maxLagSeconds }); + continue; + } + + const lagSeconds = Math.round(nowSeconds - timestamp); + if (lagSeconds > maxLagSeconds) { + stale.push({ chain: chain.name, blockNumber, lagSeconds, maxLagSeconds }); + } + } + + return stale; +} + +/** One-line, operator-readable summary of why the probe failed. */ +export function describeStaleChains(stale: StaleChain[]): string { + const parts = stale.map((s) => + s.lagSeconds === null + ? `${s.chain}: no synced block reported` + : `${s.chain}: block ${s.blockNumber ?? "?"} is ${s.lagSeconds}s old (max ${s.maxLagSeconds}s)`, + ); + return `Chain sync is stalled — ${parts.join("; ")}.`; +} diff --git a/src/api/index.ts b/src/api/index.ts index 7c98b5c..950b6cf 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -7,6 +7,8 @@ import { swaggerUI } from "@hono/swagger-ui"; import { apiRouter } from "./router"; import { gqlDocsMiddleware } from "./gql-docs"; import { OWNER_BACKFILL_TYPES } from "../utils/order-types"; +import { fetchMetricLines, parsePrometheusGauge } from "./prometheus"; +import { describeStaleChains, findStaleChains } from "./freshness"; const app = new Hono(); @@ -19,27 +21,40 @@ app.use("/graphql", graphql({ db, schema })); app.get("/healthz", (c) => c.json({ status: "ok" })); -// Readiness for blue-green promotion. Ponder's built-in /ready flips once historical -// sync reaches the tip, but OwnerBackfill drains historical orders across the following -// live blocks — so /ready alone would promote an indexer with history still filling. -// /readyz returns 200 only when BOTH Ponder is synced AND the owner backfill is complete -// (no non-deterministic historical generator still pending). Point the deployment -// readiness probe at this instead of /ready. Ponder reserves /ready, so it can't be -// shadowed — hence a distinct path. +// Readiness for blue-green promotion and for the container healthcheck. Ponder's +// built-in /ready flips once historical sync reaches the tip and then stays 200 +// forever — it says nothing about whether live indexing is still advancing, so a +// stalled indexer keeps reporting itself ready. OwnerBackfill also drains historical +// orders across the following live blocks, so /ready alone would promote an indexer +// with history still filling. /readyz returns 200 only when Ponder is synced, every +// active chain is still keeping up with the tip, and the owner backfill is complete. +// Ponder reserves /ready, so it can't be shadowed — hence a distinct path. app.get("/readyz", async (c) => { + const origin = new URL(c.req.url).origin; + // 1. Ponder historical sync complete? Reuse the built-in /ready on the same origin // (also guards the empty-DB false positive: a fresh pod with zero generators would // otherwise have a pending count of 0 and look ready before indexing anything). let synced = false; try { - const res = await fetch(`${new URL(c.req.url).origin}/ready`); + const res = await fetch(`${origin}/ready`); synced = res.status === 200; } catch { synced = false; } if (!synced) return c.text("Historical indexing is not complete.", 503); - // 2. OwnerBackfill drained? Count matches OwnerBackfill's eligibility set exactly. + // 2. Live indexing still advancing? A dead RPC subscription leaves the process + // healthy and serving while the newest synced block silently ages. + const metricLines = await fetchMetricLines(origin); + const stale = findStaleChains( + parsePrometheusGauge(metricLines, "ponder_sync_block_timestamp"), + parsePrometheusGauge(metricLines, "ponder_sync_block"), + Math.floor(Date.now() / 1000), + ); + if (stale.length > 0) return c.text(describeStaleChains(stale), 503); + + // 3. OwnerBackfill drained? Count matches OwnerBackfill's eligibility set exactly. const rows = await db .select({ pending: count() }) .from(schema.conditionalOrderGenerator) diff --git a/tests/api/freshness.test.ts b/tests/api/freshness.test.ts new file mode 100644 index 0000000..9d62768 --- /dev/null +++ b/tests/api/freshness.test.ts @@ -0,0 +1,155 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { + DEFAULT_MAX_LAG_SECONDS, + describeStaleChains, + findStaleChains, + maxLagSecondsFor, +} from "../../src/api/freshness"; +import type { ChainConfig } from "../../src/chains/types"; + +const NOW = 1_786_003_795; + +// Only the two fields findStaleChains reads. +const CHAINS = [ + { name: "mainnet", chainId: 1 }, + { name: "gnosis", chainId: 100 }, +] as unknown as ChainConfig[]; + +function gauges(entries: Record) { + return new Map(Object.entries(entries)); +} + +const FRESH_TIMESTAMPS = gauges({ mainnet: NOW - 20, gnosis: NOW - 15 }); +const BLOCK_NUMBERS = gauges({ mainnet: 25_694_617, gnosis: 47_582_895 }); + +afterEach(() => { + delete process.env.READINESS_MAX_LAG_SECONDS; + delete process.env.READINESS_MAX_LAG_SECONDS_1; + delete process.env.READINESS_MAX_LAG_SECONDS_100; +}); + +describe("maxLagSecondsFor", () => { + it("falls back to the default when nothing is configured", () => { + expect(maxLagSecondsFor(1)).toBe(DEFAULT_MAX_LAG_SECONDS); + }); + + it("prefers the per-chain override over the global one", () => { + process.env.READINESS_MAX_LAG_SECONDS = "120"; + process.env.READINESS_MAX_LAG_SECONDS_1 = "600"; + expect(maxLagSecondsFor(1)).toBe(600); + expect(maxLagSecondsFor(100)).toBe(120); + }); + + it("ignores unparseable and non-positive values", () => { + process.env.READINESS_MAX_LAG_SECONDS = "not-a-number"; + expect(maxLagSecondsFor(1)).toBe(DEFAULT_MAX_LAG_SECONDS); + process.env.READINESS_MAX_LAG_SECONDS = "0"; + expect(maxLagSecondsFor(1)).toBe(DEFAULT_MAX_LAG_SECONDS); + }); +}); + +describe("findStaleChains", () => { + it("reports nothing while every chain is near the tip", () => { + expect( + findStaleChains(FRESH_TIMESTAMPS, BLOCK_NUMBERS, NOW, CHAINS), + ).toEqual([]); + }); + + it("flags a chain whose newest block has aged past the budget", () => { + // The production failure: sync froze and the block timestamp stopped moving. + const timestamps = gauges({ mainnet: NOW - 16_670, gnosis: NOW - 10 }); + + const stale = findStaleChains(timestamps, BLOCK_NUMBERS, NOW, CHAINS); + + expect(stale).toEqual([ + { + chain: "mainnet", + blockNumber: 25_694_617, + lagSeconds: 16_670, + maxLagSeconds: DEFAULT_MAX_LAG_SECONDS, + }, + ]); + }); + + it("flags every stalled chain, not just the first", () => { + const timestamps = gauges({ mainnet: NOW - 16_670, gnosis: NOW - 16_671 }); + + expect( + findStaleChains(timestamps, BLOCK_NUMBERS, NOW, CHAINS).map((s) => s.chain), + ).toEqual(["mainnet", "gnosis"]); + }); + + it("treats a chain missing from the metrics as stale", () => { + const stale = findStaleChains( + gauges({ mainnet: NOW - 20 }), + gauges({ mainnet: 25_694_617 }), + NOW, + CHAINS, + ); + + expect(stale).toEqual([ + { + chain: "gnosis", + blockNumber: null, + lagSeconds: null, + maxLagSeconds: DEFAULT_MAX_LAG_SECONDS, + }, + ]); + }); + + it("treats an empty metrics scrape as stale rather than fresh", () => { + expect( + findStaleChains(gauges({}), gauges({}), NOW, CHAINS), + ).toHaveLength(2); + }); + + it("stays fresh right at the budget and turns stale one second past it", () => { + const atBudget = gauges({ + mainnet: NOW - DEFAULT_MAX_LAG_SECONDS, + gnosis: NOW, + }); + expect(findStaleChains(atBudget, BLOCK_NUMBERS, NOW, CHAINS)).toEqual([]); + + const pastBudget = gauges({ + mainnet: NOW - DEFAULT_MAX_LAG_SECONDS - 1, + gnosis: NOW, + }); + expect( + findStaleChains(pastBudget, BLOCK_NUMBERS, NOW, CHAINS), + ).toHaveLength(1); + }); + + it("honours a per-chain budget", () => { + process.env.READINESS_MAX_LAG_SECONDS_100 = "30"; + const timestamps = gauges({ mainnet: NOW - 60, gnosis: NOW - 60 }); + + expect( + findStaleChains(timestamps, BLOCK_NUMBERS, NOW, CHAINS).map((s) => s.chain), + ).toEqual(["gnosis"]); + }); +}); + +describe("describeStaleChains", () => { + it("names the block and its age", () => { + const message = describeStaleChains([ + { + chain: "mainnet", + blockNumber: 25_694_617, + lagSeconds: 16_670, + maxLagSeconds: 300, + }, + ]); + + expect(message).toBe( + "Chain sync is stalled — mainnet: block 25694617 is 16670s old (max 300s).", + ); + }); + + it("distinguishes a chain that reported no block at all", () => { + const message = describeStaleChains([ + { chain: "gnosis", blockNumber: null, lagSeconds: null, maxLagSeconds: 300 }, + ]); + + expect(message).toBe("Chain sync is stalled — gnosis: no synced block reported."); + }); +}); From fa36dc718c86960286d46c3bc7cdb07f4d946836 Mon Sep 17 00:00:00 2001 From: Pedro Yves Fracari Date: Thu, 6 Aug 2026 10:05:50 -0300 Subject: [PATCH 3/7] fix(deploy): point the healthcheck at /readyz and let autoheal act on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two separate gaps kept the week-long stall invisible. The health check probed /ready, which stays 200 forever after the initial sync, so it could never fail no matter how stale the data got. It now probes /readyz, which carries the freshness gate. Also pins an explicit interval and timeout rather than leaning on Docker's defaults. Docker does not restart a container for failing its health check — restart: unless-stopped only reacts to the process exiting — so even a failing check would have left the container sitting there unhealthy. The autoheal label wires it to the host's willfarrell/autoheal daemon, which restarts unhealthy labelled containers. Inert if that daemon is not running on the host. --- Dockerfile | 9 ++++++++- docker-compose.yml | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 25a7b20..335bfcf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,11 +31,18 @@ RUN pnpm install --frozen-lockfile \ USER node +# /readyz, not Ponder's /ready: /ready returns 200 forever once historical sync +# finishes, so a stalled realtime subscription still reports healthy. /readyz also +# checks that each chain's newest synced block is keeping up with wall-clock. +# Note: Docker never restarts a container for failing its healthcheck — the +# `autoheal` label on the ponder service is what turns unhealthy into a restart. HEALTHCHECK \ --start-period=24h \ --start-interval=1s \ + --interval=30s \ + --timeout=10s \ --retries=3 \ - CMD curl -f http://localhost:3000/ready || exit 1 + CMD curl -f http://localhost:3000/readyz || exit 1 EXPOSE 3000/tcp diff --git a/docker-compose.yml b/docker-compose.yml index 2070ee0..5ee46bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,14 @@ services: # (after drop) it's a no-op; on an existing schema it allows resuming from the # last checkpoint rather than wiping and starting over. PONDER_EXPERIMENTAL_DB: "platform" + # Docker restarts a container when its process exits, never when its healthcheck + # fails — a wedged-but-alive indexer would sit there unhealthy forever. The host's + # autoheal daemon (willfarrell/autoheal, AUTOHEAL_CONTAINER_LABEL=autoheal) watches + # for this label and restarts the container once HEALTHCHECK marks it unhealthy. + # Without that daemon running on the host, the label is inert and the healthcheck + # only reports; it does not recover. + labels: + autoheal: "true" ports: - "${PONDER_EXPOSED_PORT:-40000}:3000" depends_on: From 3e6dbc228c204ea2f5ef1958bb21c3b5f5461976 Mon Sep 17 00:00:00 2001 From: Pedro Yves Fracari Date: Thu, 6 Aug 2026 10:05:55 -0300 Subject: [PATCH 4/7] docs: describe the freshness gate and what actually restarts the container Records the /ready latching behaviour that hid the stall, the staleness budget and its env overrides, why the probe avoids an RPC call, and the two easy misreadings of the container health check: Docker never restarts on an unhealthy check, and the 24-hour start period is what covers a cold start. --- docs/api-reference.md | 3 ++- docs/deployment.md | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index ebe7de9..729299b 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -14,7 +14,8 @@ The default local URL is `http://localhost:42069` when using `pnpm dev`. The pro | `/docs` | GET | Swagger UI for the REST endpoints. | | `/openapi.json` | GET | OpenAPI 3.0 spec for the REST endpoints. | | `/health` | GET | Ponder built-in. Returns `200` (empty body) when the process is running. | -| `/ready` | GET | Ponder built-in. Returns `200` when initial sync is complete; `503` while still syncing. Suitable for K8s readiness probes. | +| `/ready` | GET | Ponder built-in. Returns `200` when initial sync is complete; `503` while still syncing. Latches at `200` afterwards, so it will not detect a stalled indexer — prefer `/readyz`. | +| `/readyz` | GET | Application-level readiness. Returns `200` only when Ponder is synced, every active chain's newest block is within `READINESS_MAX_LAG_SECONDS` of now, and the owner backfill has drained. `503` with a plain-text reason otherwise. This is the promotion probe and the container health check. | | `/healthz` | GET | Application-level. Returns `{ "status": "ok" }` when the server is up. Does not reflect indexer sync progress. | | `/status` | GET | Sync progress per chain. Returns current indexed block, latest chain block, and a completion percentage. Useful for monitoring backfill progress. | | `/metrics` | GET | Prometheus metrics. Exposes Ponder internals (block lag, handler latency, RPC call counts). | diff --git a/docs/deployment.md b/docs/deployment.md index adfc6b3..6b1f548 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -89,7 +89,7 @@ The deploy services (`postgres-deploy` and `ponder`) live in the root `docker-co docker compose --profile deploy up -d ``` -The `Dockerfile` in the project root builds the Ponder image: two-stage Node 22 Alpine, installs dependencies with `--frozen-lockfile`, exposes port 3000, runs `pnpm start`. The health check hits `/ready` with a 24-hour start period (initial sync takes hours). +The `Dockerfile` in the project root builds the Ponder image: two-stage Node 22 Alpine, installs dependencies with `--frozen-lockfile`, exposes port 3000, runs `pnpm start`. The health check hits `/readyz` with a 24-hour start period (initial sync takes hours). ### Kubernetes Probes @@ -99,9 +99,11 @@ The indexer exposes two health endpoints with distinct semantics: |----------|----------|-----------------| | `/health` | **Liveness** — is the process alive? | Always, once the server starts | | `/ready` | Ponder sync — has it reached the chain tip? | Only when historical sync is complete | -| `/readyz` | **Readiness** — synced **and** owner backfill complete | Ponder synced AND no non-deterministic historical generator still pending | +| `/readyz` | **Readiness** — synced, keeping up, **and** owner backfill complete | Ponder synced AND every active chain within its staleness budget AND no non-deterministic historical generator still pending | -Use **`/readyz`** as the readiness/promotion probe. Ponder's built-in `/ready` flips as soon as historical sync reaches the tip, and `OwnerBackfillLive` then drains historical discrete orders across the following live-sync blocks. `/readyz` waits for both: it returns 200 only once Ponder is synced **and** `COUNT(historyBackfilled = false) = 0` (it internally checks `/ready` first, so it also can't false-positive on an empty fresh DB). Expect it to report pending during the drain, which begins after `/ready` flips. Ponder reserves the `/ready` path, so `/readyz` is a distinct endpoint served by the app. +Use **`/readyz`** as the readiness/promotion probe. Ponder's built-in `/ready` flips as soon as historical sync reaches the tip, and `OwnerBackfillLive` then drains historical discrete orders across the following live-sync blocks. `/readyz` waits for all three conditions: Ponder synced, no chain lagging the tip, and `COUNT(historyBackfilled = false) = 0` (it internally checks `/ready` first, so it also can't false-positive on an empty fresh DB). Expect it to report pending during the drain, which begins after `/ready` flips. Ponder reserves the `/ready` path, so `/readyz` is a distinct endpoint served by the app. + +The staleness check reads `ponder_sync_block_timestamp` per chain from `/metrics` and compares it against wall-clock time. Anything older than `READINESS_MAX_LAG_SECONDS` (default 300, overridable per chain with a numeric chain-id suffix) makes `/readyz` return 503 naming the chain, its newest block, and the measured lag. It deliberately does not call the RPC for the current head: a probe that depends on the RPC turns an RPC outage into a restart loop, and the block timestamp already carries enough to spot a stalled sync. The trade-off is that a genuine chain halt reads as staleness. Map these to different K8s probe types. The specific timing values (`periodSeconds`, `failureThreshold`, `initialDelaySeconds`) depend on your cluster's SLOs; what matters is which path and port to use: @@ -126,7 +128,11 @@ readinessProbe: A pod in `NotReady` state is not killed — it is simply removed from load-balancer rotation. On a cold start (no existing database), the pod will be `NotReady` for the duration of the historical backfill (hours). That is expected: the old pod (if any) keeps serving traffic during this window, and once the new pod catches up, K8s starts routing to it. -The Docker Compose health check uses `/ready` with a 24-hour start period as a pragmatic fallback for single-container deployments, not as a K8s-style probe. +The container health check (declared in the `Dockerfile`) uses `/readyz` with a 24-hour start period as a pragmatic fallback for single-container deployments, not as a K8s-style probe. Two things about it are easy to get wrong: + +Docker never restarts a container because its health check fails — `restart: unless-stopped` only reacts to the process exiting. A wedged-but-alive indexer stays up, marked unhealthy, indefinitely. The `autoheal: "true"` label on the ponder service is what closes that loop: the host's `willfarrell/autoheal` daemon (started with `AUTOHEAL_CONTAINER_LABEL=autoheal`) polls for unhealthy labelled containers and restarts them. If that daemon is not running on the host, the label does nothing and the health check only reports. + +The 24-hour start period exists because a cold start legitimately fails `/readyz` for hours. During the start period a failing check does not mark the container unhealthy; the first success ends it. The owner-backfill gate cannot flap afterwards — generators created during live sync are written with `historyBackfilled = true`, so the pending count never climbs back above zero once the initial drain finishes. ### Structured Logging @@ -192,7 +198,9 @@ A reindex that reuses an existing `ponder_sync` cache (same chain, same start bl `GET /ready` (Ponder built-in) returns `200` when Ponder has processed all historical blocks up to the tip and the live indexer is running. It does **not** guarantee historical discrete-order data is complete — `OwnerBackfillLive` drains that across live blocks after the tip. -`GET /readyz` (app) returns `200` only when Ponder is synced **and** the owner backfill has finished (no non-deterministic historical generator with `historyBackfilled = false`). This is the promotion gate: it guarantees a newly-promoted pod has the full historical discrete-order set, so blue-green promotion never drops a complete pod for one that's still filling. It returns `503` (with the pending count) while the drain is in progress. +`GET /readyz` (app) returns `200` only when Ponder is synced, no active chain has fallen behind the tip, **and** the owner backfill has finished (no non-deterministic historical generator with `historyBackfilled = false`). This is the promotion gate: it guarantees a newly-promoted pod has the full historical discrete-order set, so blue-green promotion never drops a complete pod for one that's still filling. It returns `503` (with the pending count) while the drain is in progress. + +The staleness condition also makes `/readyz` useful after startup, which `/ready` is not. `/ready` latches at 200 once historical sync completes and never goes back, so an indexer whose realtime subscription dies keeps advertising itself as ready while its data silently ages. That happened in production: a dRPC WebSocket stopped delivering `newHeads` without closing the connection, Ponder logged one "No new block received within expected time" warning and then sat idle, and `/ready` plus the container health check both stayed green for a week. During backfill both return `503`. GraphQL queries are still available but data is incomplete (generators and transactions accumulate; discrete orders fill in as live sync progresses). From aa202293a9f5e3a9c600ab020c6fb8846b6e495e Mon Sep 17 00:00:00 2001 From: Pedro Yves Fracari Date: Thu, 6 Aug 2026 10:28:38 -0300 Subject: [PATCH 5/7] docs(deployment): record the API/indexer split option and its probe caveat Nothing in docs/ mentioned that Ponder can run the HTTP server separately from the indexer, so a team taking over operations had no way to know the option exists. Documents ponder serve, the four constraints read off 0.16.6, and the fact that it has not been run here. Calls out the interaction with the freshness gate: it reads an in-memory gauge that only the indexing process sets, so an API-only container would fail /readyz forever and autoheal would restart it in a loop. Records _ponder_checkpoint as the DB-backed source to switch to, with the internal-table caveat. --- docs/deployment.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/deployment.md b/docs/deployment.md index 6b1f548..dc4408d 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -134,6 +134,27 @@ Docker never restarts a container because its health check fails — `restart: u The 24-hour start period exists because a cold start legitimately fails `/readyz` for hours. During the start period a failing check does not mark the container unhealthy; the first success ends it. The owner-backfill gate cannot flap afterwards — generators created during live sync are written with `historyBackfilled = true`, so the pending count never climbs back above zero once the initial drain finishes. +### Splitting the API from the Indexer + +Today one container does both jobs: `pnpm start` runs `ponder start`, which indexes and serves HTTP in a single process. Every indexer restart therefore takes the API down with it — including the automatic ones autoheal performs when the sync stalls. Ponder supports separating the two, and this section records what that would involve. **It has not been run in this project** — treat it as a researched option, not a tested path. + +`ponder serve` starts the HTTP server without the indexer. Same image, different command: one container keeps running `ponder start` (indexing, and incidentally still serving on its own port), a second runs `ponder serve` against the same database and schema, and the public route points at the second one. Restarting the indexer then leaves API traffic untouched. + +Four constraints, read off the installed Ponder 0.16.6: + +| Aspect | Behaviour under `ponder serve` | +|--------|-------------------------------| +| Database | Postgres only — it exits with an error on PGlite. | +| Schema | Must already exist. It logs `Schema does not exist` and exits 1, so the API container cannot start before the indexer has created the schema. Plan the startup ordering, or let it restart until the schema appears. | +| `/ready` | Works normally. It reads the `is_ready` flag from the database rather than from process state. | +| `ponder_sync_*` metrics | Absent. They are in-memory gauges set only by the indexing runtime (`runtime/realtime.js`, `runtime/historical.js`), which serve mode never runs. | + +The RPC diagnostic is skipped in serve mode, but `ponder.config.ts` still executes, so give both containers the same environment rather than trimming the RPC variables from the API one. + +That last table row is the trap. `/readyz`'s freshness gate reads `ponder_sync_block_timestamp` from `/metrics`, which only exists in the indexing process. On an API-only container every chain would read as "no synced block reported", `/readyz` would return 503 permanently, and the health check plus autoheal would restart that container in a loop. **If you split, move the freshness gate off the metrics endpoint** and onto `_ponder_checkpoint`, a table in the app schema holding `chain_name`, `chain_id`, and `latest_checkpoint` for each chain. The checkpoint is a fixed-width encoding whose leading 10 digits are the block timestamp, with the block number in a later field. Both processes can read it, so the same probe then works in either topology, and it removes the self-HTTP call to `/metrics`. The cost is a dependency on an internal table — the `_ponder_` prefix marks it as Ponder's own, and its encoding may change across versions, whereas metric names are the more stable surface. + +Be clear about what this does and does not buy. It keeps the API available across indexer restarts. It does nothing about staleness: during a stall the API stays up and serves stale data, which is the failure mode described under `/ready` vs `/readyz` Semantics. Splitting decouples availability from restarts; the readiness gate is what tells you the data has gone cold. + ### Structured Logging `pnpm start` runs with `--log-format json`, which makes both Ponder's internal log lines and the handler log lines emit newline-delimited JSON. Each handler log line includes structured fields (e.g. `chainId`, `block`) enabling log aggregators (Datadog, CloudWatch, Loki) to filter and alert by chain. From 5b59c21f5db7209385c5817fba8c5a70e4fe1720 Mon Sep 17 00:00:00 2001 From: Pedro Yves Fracari Date: Thu, 6 Aug 2026 10:57:23 -0300 Subject: [PATCH 6/7] refactor(api): read chain freshness from /status instead of /metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The freshness gate parsed ponder_sync_block_timestamp out of the Prometheus text format. Those gauges live in the indexing process's memory, which pinned the probe to a process that also has to be the one serving HTTP. Ponder's /status exposes the same information from the database — it decodes the _ponder_checkpoint table into a block number and timestamp per chain — so the check now reads committed state instead. Concretely this means /readyz stays correct if the API is ever split from the indexer with ponder serve, drops the Prometheus text parsing, and reads a documented JSON endpoint rather than internal gauge names. Also restores the Prometheus parser to sync-progress.ts. It was extracted for the freshness gate to share; with that gone, sync-progress is its only consumer again and the extra module earned nothing. --- docs/deployment.md | 13 +-- src/api/endpoints/sync-progress.ts | 30 ++++++- src/api/freshness.ts | 46 +++++++--- src/api/index.ts | 14 +-- src/api/prometheus.ts | 29 ------ tests/api/freshness.test.ts | 139 +++++++++++++++++++---------- 6 files changed, 172 insertions(+), 99 deletions(-) delete mode 100644 src/api/prometheus.ts diff --git a/docs/deployment.md b/docs/deployment.md index dc4408d..fc585e1 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -103,7 +103,7 @@ The indexer exposes two health endpoints with distinct semantics: Use **`/readyz`** as the readiness/promotion probe. Ponder's built-in `/ready` flips as soon as historical sync reaches the tip, and `OwnerBackfillLive` then drains historical discrete orders across the following live-sync blocks. `/readyz` waits for all three conditions: Ponder synced, no chain lagging the tip, and `COUNT(historyBackfilled = false) = 0` (it internally checks `/ready` first, so it also can't false-positive on an empty fresh DB). Expect it to report pending during the drain, which begins after `/ready` flips. Ponder reserves the `/ready` path, so `/readyz` is a distinct endpoint served by the app. -The staleness check reads `ponder_sync_block_timestamp` per chain from `/metrics` and compares it against wall-clock time. Anything older than `READINESS_MAX_LAG_SECONDS` (default 300, overridable per chain with a numeric chain-id suffix) makes `/readyz` return 503 naming the chain, its newest block, and the measured lag. It deliberately does not call the RPC for the current head: a probe that depends on the RPC turns an RPC outage into a restart loop, and the block timestamp already carries enough to spot a stalled sync. The trade-off is that a genuine chain halt reads as staleness. +The staleness check reads each chain's newest indexed block from Ponder's `/status` and compares its timestamp against wall-clock time. Anything older than `READINESS_MAX_LAG_SECONDS` (default 300, overridable per chain with a numeric chain-id suffix) makes `/readyz` return 503 naming the chain, its newest block, and the measured lag. It deliberately does not call the RPC for the current head, because it wants to be robust to RPC outages. `/status` decodes the `_ponder_checkpoint` table, so the check reads committed database state rather than the indexing process's memory — which is what keeps it correct if the API is ever run separately from the indexer. Map these to different K8s probe types. The specific timing values (`periodSeconds`, `failureThreshold`, `initialDelaySeconds`) depend on your cluster's SLOs; what matters is which path and port to use: @@ -130,13 +130,13 @@ A pod in `NotReady` state is not killed — it is simply removed from load-balan The container health check (declared in the `Dockerfile`) uses `/readyz` with a 24-hour start period as a pragmatic fallback for single-container deployments, not as a K8s-style probe. Two things about it are easy to get wrong: -Docker never restarts a container because its health check fails — `restart: unless-stopped` only reacts to the process exiting. A wedged-but-alive indexer stays up, marked unhealthy, indefinitely. The `autoheal: "true"` label on the ponder service is what closes that loop: the host's `willfarrell/autoheal` daemon (started with `AUTOHEAL_CONTAINER_LABEL=autoheal`) polls for unhealthy labelled containers and restarts them. If that daemon is not running on the host, the label does nothing and the health check only reports. +Docker never restarts a container because its health check fails — `restart: unless-stopped` only reacts to the process exiting. A wedged-but-alive indexer stays up. The host's `willfarrell/autoheal` daemon (started with `AUTOHEAL_CONTAINER_LABEL=autoheal`) polls for unhealthy labelled containers and restarts them. -The 24-hour start period exists because a cold start legitimately fails `/readyz` for hours. During the start period a failing check does not mark the container unhealthy; the first success ends it. The owner-backfill gate cannot flap afterwards — generators created during live sync are written with `historyBackfilled = true`, so the pending count never climbs back above zero once the initial drain finishes. +The 24-hour start period exists because a cold start legitimately fails `/readyz` for hours. During the start period a failing check does not mark the container unhealthy; the first success ends it. ### Splitting the API from the Indexer -Today one container does both jobs: `pnpm start` runs `ponder start`, which indexes and serves HTTP in a single process. Every indexer restart therefore takes the API down with it — including the automatic ones autoheal performs when the sync stalls. Ponder supports separating the two, and this section records what that would involve. **It has not been run in this project** — treat it as a researched option, not a tested path. +Today one container does both jobs: `pnpm start` runs `ponder start`, which indexes and serves HTTP in a single process. Every indexer restart therefore takes the API down with it — including the automatic ones autoheal performs when the sync stalls. Ponder supports separating the two. `ponder serve` starts the HTTP server without the indexer. Same image, different command: one container keeps running `ponder start` (indexing, and incidentally still serving on its own port), a second runs `ponder serve` against the same database and schema, and the public route points at the second one. Restarting the indexer then leaves API traffic untouched. @@ -147,11 +147,12 @@ Four constraints, read off the installed Ponder 0.16.6: | Database | Postgres only — it exits with an error on PGlite. | | Schema | Must already exist. It logs `Schema does not exist` and exits 1, so the API container cannot start before the indexer has created the schema. Plan the startup ordering, or let it restart until the schema appears. | | `/ready` | Works normally. It reads the `is_ready` flag from the database rather than from process state. | -| `ponder_sync_*` metrics | Absent. They are in-memory gauges set only by the indexing runtime (`runtime/realtime.js`, `runtime/historical.js`), which serve mode never runs. | +| `/status` | Works normally. It decodes the `_ponder_checkpoint` table, so it reports the newest indexed block per chain regardless of which process is asking. | +| `ponder_sync_*` metrics | Absent. They are in-memory gauges set only by the indexing runtime (`runtime/realtime.js`, `runtime/historical.js`), which serve mode never runs. Nothing in this repo depends on them; `/api/sync-progress` reads the `ponder_historical_*` gauges and would report empty on an API-only container. | The RPC diagnostic is skipped in serve mode, but `ponder.config.ts` still executes, so give both containers the same environment rather than trimming the RPC variables from the API one. -That last table row is the trap. `/readyz`'s freshness gate reads `ponder_sync_block_timestamp` from `/metrics`, which only exists in the indexing process. On an API-only container every chain would read as "no synced block reported", `/readyz` would return 503 permanently, and the health check plus autoheal would restart that container in a loop. **If you split, move the freshness gate off the metrics endpoint** and onto `_ponder_checkpoint`, a table in the app schema holding `chain_name`, `chain_id`, and `latest_checkpoint` for each chain. The checkpoint is a fixed-width encoding whose leading 10 digits are the block timestamp, with the block number in a later field. Both processes can read it, so the same probe then works in either topology, and it removes the self-HTTP call to `/metrics`. The cost is a dependency on an internal table — the `_ponder_` prefix marks it as Ponder's own, and its encoding may change across versions, whereas metric names are the more stable surface. +`/readyz` works in either topology. Both conditions it evaluates — Ponder's `/ready` and the per-chain freshness check via `/status` — read committed database state, so an API-only container reports on the indexer's progress correctly rather than on its own idle process. This is the reason the freshness gate reads `/status` and not `ponder_sync_block_timestamp` from `/metrics`: the metrics route would have pinned the probe to the indexing process and made an API-only container fail readiness permanently. Be clear about what this does and does not buy. It keeps the API available across indexer restarts. It does nothing about staleness: during a stall the API stays up and serves stale data, which is the failure mode described under `/ready` vs `/readyz` Semantics. Splitting decouples availability from restarts; the readiness gate is what tells you the data has gone cold. diff --git a/src/api/endpoints/sync-progress.ts b/src/api/endpoints/sync-progress.ts index b38ac69..a856b12 100644 --- a/src/api/endpoints/sync-progress.ts +++ b/src/api/endpoints/sync-progress.ts @@ -1,11 +1,37 @@ import type { RouteHandler } from "@hono/zod-openapi"; import type { syncProgressRoute } from "../routes"; -import { fetchMetricLines, parsePrometheusGauge } from "../prometheus"; + +// Prometheus text-format parser for a single gauge metric. +// Matches lines like: metric_name{label="value"} 123 +const GAUGE_RE = /^(\w+)\{([^}]*)\}\s+([\d.]+)/; + +function parsePrometheusGauge( + lines: string[], + metricName: string, +): Map { + const result = new Map(); + for (const line of lines) { + if (!line.startsWith(metricName + "{")) continue; + const m = GAUGE_RE.exec(line); + if (!m) continue; + const labels = m[2] as string; + const value = Number(m[3]); + // Extract chain label value + const chainMatch = /chain="([^"]+)"/.exec(labels); + if (chainMatch) result.set(chainMatch[1] as string, value); + } + return result; +} export const syncProgressHandler: RouteHandler = async (c) => { // Resolve /metrics relative to the current request so this works on any port. - const lines = await fetchMetricLines(new URL(c.req.url).origin); + const origin = new URL(c.req.url).origin; + const metricsText = await fetch(`${origin}/metrics`) + .then((r) => r.text()) + .catch(() => ""); + + const lines = metricsText.split("\n"); const total = parsePrometheusGauge(lines, "ponder_historical_total_blocks"); const completed = parsePrometheusGauge( diff --git a/src/api/freshness.ts b/src/api/freshness.ts index 8b09388..4d02f57 100644 --- a/src/api/freshness.ts +++ b/src/api/freshness.ts @@ -2,7 +2,7 @@ import { ACTIVE_CHAINS } from "../chains"; import type { ChainConfig } from "../chains/types"; /** - * How far the newest synced block may fall behind wall-clock before a chain is + * How far the newest indexed block may fall behind wall-clock before a chain is * considered stalled. Generous by default: the live block handlers * (OrderDiscoveryPoller, CandidateConfirmer, …) each add seconds of latency per * firing, so normal operation already lags tens of seconds behind the tip. @@ -21,9 +21,33 @@ export function maxLagSecondsFor(chainId: number): number { return DEFAULT_MAX_LAG_SECONDS; } +/** Ponder's /status payload, keyed by chain name. */ +export type ChainStatus = Record< + string, + { id?: number; block?: { number?: number; timestamp?: number } | null } | null +>; + +/** + * Read the newest indexed block per chain from Ponder's /status. + * + * /status decodes the `_ponder_checkpoint` table, so it reflects committed + * database state rather than the indexing process's memory. That matters for + * two reasons: it stays correct if the API is ever run separately from the + * indexer (`ponder serve`), and it avoids parsing the Prometheus text format. + */ +export async function fetchChainStatus(origin: string): Promise { + try { + const res = await fetch(`${origin}/status`); + if (!res.ok) return {}; + return (await res.json()) as ChainStatus; + } catch { + return {}; + } +} + export type StaleChain = { chain: string; - /** Newest synced block number, or null when the chain reports no block at all. */ + /** Newest indexed block number, or null when the chain reports no block at all. */ blockNumber: number | null; /** Seconds between that block's timestamp and now, or null when unknown. */ lagSeconds: number | null; @@ -31,19 +55,18 @@ export type StaleChain = { }; /** - * Compare each active chain's newest synced block against wall-clock time. + * Compare each active chain's newest indexed block against wall-clock time. * * Wall-clock rather than a fresh `eth_blockNumber` call on purpose: a readiness * probe that depends on the RPC turns an RPC outage into a restart loop, and the * block timestamp already carries everything needed to spot a stalled sync. - * The trade-off is that a genuine chain halt reads as staleness here. + * The trade-off is that a genuine chain halt reads as staleness. * - * A chain missing from the metrics counts as stale — absent data is not evidence + * A chain missing from the payload counts as stale — absent data is not evidence * of freshness. */ export function findStaleChains( - blockTimestamps: Map, - blockNumbers: Map, + status: ChainStatus, nowSeconds: number, chains: ChainConfig[] = ACTIVE_CHAINS, ): StaleChain[] { @@ -51,10 +74,11 @@ export function findStaleChains( for (const chain of chains) { const maxLagSeconds = maxLagSecondsFor(chain.chainId); - const blockNumber = blockNumbers.get(chain.name) ?? null; - const timestamp = blockTimestamps.get(chain.name); + const block = status[chain.name]?.block; + const blockNumber = typeof block?.number === "number" ? block.number : null; + const timestamp = block?.timestamp; - if (timestamp === undefined || timestamp <= 0) { + if (typeof timestamp !== "number" || timestamp <= 0) { stale.push({ chain: chain.name, blockNumber, lagSeconds: null, maxLagSeconds }); continue; } @@ -72,7 +96,7 @@ export function findStaleChains( export function describeStaleChains(stale: StaleChain[]): string { const parts = stale.map((s) => s.lagSeconds === null - ? `${s.chain}: no synced block reported` + ? `${s.chain}: no indexed block reported` : `${s.chain}: block ${s.blockNumber ?? "?"} is ${s.lagSeconds}s old (max ${s.maxLagSeconds}s)`, ); return `Chain sync is stalled — ${parts.join("; ")}.`; diff --git a/src/api/index.ts b/src/api/index.ts index 950b6cf..b0387cb 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -7,8 +7,11 @@ import { swaggerUI } from "@hono/swagger-ui"; import { apiRouter } from "./router"; import { gqlDocsMiddleware } from "./gql-docs"; import { OWNER_BACKFILL_TYPES } from "../utils/order-types"; -import { fetchMetricLines, parsePrometheusGauge } from "./prometheus"; -import { describeStaleChains, findStaleChains } from "./freshness"; +import { + describeStaleChains, + fetchChainStatus, + findStaleChains, +} from "./freshness"; const app = new Hono(); @@ -45,11 +48,10 @@ app.get("/readyz", async (c) => { if (!synced) return c.text("Historical indexing is not complete.", 503); // 2. Live indexing still advancing? A dead RPC subscription leaves the process - // healthy and serving while the newest synced block silently ages. - const metricLines = await fetchMetricLines(origin); + // healthy and serving while the newest indexed block silently ages. /status + // decodes _ponder_checkpoint, so this reads committed database state. const stale = findStaleChains( - parsePrometheusGauge(metricLines, "ponder_sync_block_timestamp"), - parsePrometheusGauge(metricLines, "ponder_sync_block"), + await fetchChainStatus(origin), Math.floor(Date.now() / 1000), ); if (stale.length > 0) return c.text(describeStaleChains(stale), 503); diff --git a/src/api/prometheus.ts b/src/api/prometheus.ts deleted file mode 100644 index f941a89..0000000 --- a/src/api/prometheus.ts +++ /dev/null @@ -1,29 +0,0 @@ -// Prometheus text-format parser for a single gauge metric. -// Matches lines like: metric_name{label="value"} 123 -const GAUGE_RE = /^(\w+)\{([^}]*)\}\s+([\d.]+)/; - -/** Parse one gauge from Ponder's /metrics output into a map keyed by chain label. */ -export function parsePrometheusGauge( - lines: string[], - metricName: string, -): Map { - const result = new Map(); - for (const line of lines) { - if (!line.startsWith(metricName + "{")) continue; - const m = GAUGE_RE.exec(line); - if (!m) continue; - const labels = m[2] as string; - const value = Number(m[3]); - const chainMatch = /chain="([^"]+)"/.exec(labels); - if (chainMatch) result.set(chainMatch[1] as string, value); - } - return result; -} - -/** Fetch Ponder's /metrics on the same origin and split it into lines. */ -export async function fetchMetricLines(origin: string): Promise { - const text = await fetch(`${origin}/metrics`) - .then((r) => r.text()) - .catch(() => ""); - return text.split("\n"); -} diff --git a/tests/api/freshness.test.ts b/tests/api/freshness.test.ts index 9d62768..487bc06 100644 --- a/tests/api/freshness.test.ts +++ b/tests/api/freshness.test.ts @@ -1,13 +1,15 @@ -import { describe, it, expect, afterEach } from "vitest"; +import { describe, it, expect, vi, afterEach } from "vitest"; import { DEFAULT_MAX_LAG_SECONDS, describeStaleChains, + fetchChainStatus, findStaleChains, maxLagSecondsFor, + type ChainStatus, } from "../../src/api/freshness"; import type { ChainConfig } from "../../src/chains/types"; -const NOW = 1_786_003_795; +const NOW = 1_786_024_484; // Only the two fields findStaleChains reads. const CHAINS = [ @@ -15,17 +17,25 @@ const CHAINS = [ { name: "gnosis", chainId: 100 }, ] as unknown as ChainConfig[]; -function gauges(entries: Record) { - return new Map(Object.entries(entries)); +/** Shape copied from a live /status response. */ +function status( + entries: Record, +): ChainStatus { + return Object.fromEntries( + Object.entries(entries).map(([chain, block]) => [chain, { block }]), + ); } -const FRESH_TIMESTAMPS = gauges({ mainnet: NOW - 20, gnosis: NOW - 15 }); -const BLOCK_NUMBERS = gauges({ mainnet: 25_694_617, gnosis: 47_582_895 }); +const FRESH = status({ + mainnet: { number: 25_696_339, timestamp: NOW - 21 }, + gnosis: { number: 47_586_912, timestamp: NOW - 14 }, +}); afterEach(() => { delete process.env.READINESS_MAX_LAG_SECONDS; delete process.env.READINESS_MAX_LAG_SECONDS_1; delete process.env.READINESS_MAX_LAG_SECONDS_100; + vi.unstubAllGlobals(); }); describe("maxLagSecondsFor", () => { @@ -48,20 +58,53 @@ describe("maxLagSecondsFor", () => { }); }); +describe("fetchChainStatus", () => { + it("returns the parsed payload", async () => { + const body = { + mainnet: { id: 1, block: { number: 25_696_339, timestamp: NOW - 21 } }, + }; + vi.stubGlobal( + "fetch", + vi.fn(() => + Promise.resolve({ ok: true, json: () => Promise.resolve(body) } as Response), + ), + ); + + await expect(fetchChainStatus("http://localhost:3000")).resolves.toEqual(body); + }); + + it("returns an empty payload on a non-200, so every chain reads as stale", async () => { + vi.stubGlobal( + "fetch", + vi.fn(() => Promise.resolve({ ok: false } as Response)), + ); + + await expect(fetchChainStatus("http://localhost:3000")).resolves.toEqual({}); + }); + + it("returns an empty payload when the request throws", async () => { + vi.stubGlobal( + "fetch", + vi.fn(() => Promise.reject(new Error("ECONNREFUSED"))), + ); + + await expect(fetchChainStatus("http://localhost:3000")).resolves.toEqual({}); + }); +}); + describe("findStaleChains", () => { it("reports nothing while every chain is near the tip", () => { - expect( - findStaleChains(FRESH_TIMESTAMPS, BLOCK_NUMBERS, NOW, CHAINS), - ).toEqual([]); + expect(findStaleChains(FRESH, NOW, CHAINS)).toEqual([]); }); it("flags a chain whose newest block has aged past the budget", () => { - // The production failure: sync froze and the block timestamp stopped moving. - const timestamps = gauges({ mainnet: NOW - 16_670, gnosis: NOW - 10 }); - - const stale = findStaleChains(timestamps, BLOCK_NUMBERS, NOW, CHAINS); + // The production failure: sync froze and the checkpoint stopped moving. + const frozen = status({ + mainnet: { number: 25_694_617, timestamp: NOW - 16_670 }, + gnosis: { number: 47_586_912, timestamp: NOW - 10 }, + }); - expect(stale).toEqual([ + expect(findStaleChains(frozen, NOW, CHAINS)).toEqual([ { chain: "mainnet", blockNumber: 25_694_617, @@ -72,22 +115,23 @@ describe("findStaleChains", () => { }); it("flags every stalled chain, not just the first", () => { - const timestamps = gauges({ mainnet: NOW - 16_670, gnosis: NOW - 16_671 }); + const frozen = status({ + mainnet: { number: 25_694_617, timestamp: NOW - 16_670 }, + gnosis: { number: 47_582_895, timestamp: NOW - 16_671 }, + }); - expect( - findStaleChains(timestamps, BLOCK_NUMBERS, NOW, CHAINS).map((s) => s.chain), - ).toEqual(["mainnet", "gnosis"]); + expect(findStaleChains(frozen, NOW, CHAINS).map((s) => s.chain)).toEqual([ + "mainnet", + "gnosis", + ]); }); - it("treats a chain missing from the metrics as stale", () => { - const stale = findStaleChains( - gauges({ mainnet: NOW - 20 }), - gauges({ mainnet: 25_694_617 }), - NOW, - CHAINS, - ); + it("treats a chain missing from the payload as stale", () => { + const partial = status({ + mainnet: { number: 25_696_339, timestamp: NOW - 21 }, + }); - expect(stale).toEqual([ + expect(findStaleChains(partial, NOW, CHAINS)).toEqual([ { chain: "gnosis", blockNumber: null, @@ -97,35 +141,40 @@ describe("findStaleChains", () => { ]); }); - it("treats an empty metrics scrape as stale rather than fresh", () => { - expect( - findStaleChains(gauges({}), gauges({}), NOW, CHAINS), - ).toHaveLength(2); + it("treats an unreachable /status as stale rather than fresh", () => { + expect(findStaleChains({}, NOW, CHAINS)).toHaveLength(2); + }); + + it("treats a malformed entry as stale", () => { + const malformed = { mainnet: { block: null }, gnosis: null } as ChainStatus; + + expect(findStaleChains(malformed, NOW, CHAINS)).toHaveLength(2); }); it("stays fresh right at the budget and turns stale one second past it", () => { - const atBudget = gauges({ - mainnet: NOW - DEFAULT_MAX_LAG_SECONDS, - gnosis: NOW, + const atBudget = status({ + mainnet: { number: 1, timestamp: NOW - DEFAULT_MAX_LAG_SECONDS }, + gnosis: { number: 2, timestamp: NOW }, }); - expect(findStaleChains(atBudget, BLOCK_NUMBERS, NOW, CHAINS)).toEqual([]); + expect(findStaleChains(atBudget, NOW, CHAINS)).toEqual([]); - const pastBudget = gauges({ - mainnet: NOW - DEFAULT_MAX_LAG_SECONDS - 1, - gnosis: NOW, + const pastBudget = status({ + mainnet: { number: 1, timestamp: NOW - DEFAULT_MAX_LAG_SECONDS - 1 }, + gnosis: { number: 2, timestamp: NOW }, }); - expect( - findStaleChains(pastBudget, BLOCK_NUMBERS, NOW, CHAINS), - ).toHaveLength(1); + expect(findStaleChains(pastBudget, NOW, CHAINS)).toHaveLength(1); }); it("honours a per-chain budget", () => { process.env.READINESS_MAX_LAG_SECONDS_100 = "30"; - const timestamps = gauges({ mainnet: NOW - 60, gnosis: NOW - 60 }); + const lagging = status({ + mainnet: { number: 1, timestamp: NOW - 60 }, + gnosis: { number: 2, timestamp: NOW - 60 }, + }); - expect( - findStaleChains(timestamps, BLOCK_NUMBERS, NOW, CHAINS).map((s) => s.chain), - ).toEqual(["gnosis"]); + expect(findStaleChains(lagging, NOW, CHAINS).map((s) => s.chain)).toEqual([ + "gnosis", + ]); }); }); @@ -150,6 +199,6 @@ describe("describeStaleChains", () => { { chain: "gnosis", blockNumber: null, lagSeconds: null, maxLagSeconds: 300 }, ]); - expect(message).toBe("Chain sync is stalled — gnosis: no synced block reported."); + expect(message).toBe("Chain sync is stalled — gnosis: no indexed block reported."); }); }); From 8b84bffcb9a79470b5fd3c18a1fda9b36251b221 Mon Sep 17 00:00:00 2001 From: Pedro Yves Fracari Date: Thu, 6 Aug 2026 11:22:44 -0300 Subject: [PATCH 7/7] docs: tighten the readiness and split-deployment sections Trims the endpoint tables and drops the restated rationale, keeping the constraints themselves. Notes that splitting the API from the indexer also enables horizontal scaling on the API side. --- docs/api-reference.md | 2 +- docs/deployment.md | 26 +++++++++++--------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 729299b..11bb64e 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -15,7 +15,7 @@ The default local URL is `http://localhost:42069` when using `pnpm dev`. The pro | `/openapi.json` | GET | OpenAPI 3.0 spec for the REST endpoints. | | `/health` | GET | Ponder built-in. Returns `200` (empty body) when the process is running. | | `/ready` | GET | Ponder built-in. Returns `200` when initial sync is complete; `503` while still syncing. Latches at `200` afterwards, so it will not detect a stalled indexer — prefer `/readyz`. | -| `/readyz` | GET | Application-level readiness. Returns `200` only when Ponder is synced, every active chain's newest block is within `READINESS_MAX_LAG_SECONDS` of now, and the owner backfill has drained. `503` with a plain-text reason otherwise. This is the promotion probe and the container health check. | +| `/readyz` | GET | Application-level readiness. Returns `200` only when Ponder is synced, every active chain's newest block is within `READINESS_MAX_LAG_SECONDS` of now, and the owner backfill has drained. `503` with a plain-text reason otherwise. | | `/healthz` | GET | Application-level. Returns `{ "status": "ok" }` when the server is up. Does not reflect indexer sync progress. | | `/status` | GET | Sync progress per chain. Returns current indexed block, latest chain block, and a completion percentage. Useful for monitoring backfill progress. | | `/metrics` | GET | Prometheus metrics. Exposes Ponder internals (block lag, handler latency, RPC call counts). | diff --git a/docs/deployment.md b/docs/deployment.md index fc585e1..160a428 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -101,9 +101,9 @@ The indexer exposes two health endpoints with distinct semantics: | `/ready` | Ponder sync — has it reached the chain tip? | Only when historical sync is complete | | `/readyz` | **Readiness** — synced, keeping up, **and** owner backfill complete | Ponder synced AND every active chain within its staleness budget AND no non-deterministic historical generator still pending | -Use **`/readyz`** as the readiness/promotion probe. Ponder's built-in `/ready` flips as soon as historical sync reaches the tip, and `OwnerBackfillLive` then drains historical discrete orders across the following live-sync blocks. `/readyz` waits for all three conditions: Ponder synced, no chain lagging the tip, and `COUNT(historyBackfilled = false) = 0` (it internally checks `/ready` first, so it also can't false-positive on an empty fresh DB). Expect it to report pending during the drain, which begins after `/ready` flips. Ponder reserves the `/ready` path, so `/readyz` is a distinct endpoint served by the app. +Use **`/readyz`** as the readiness/promotion probe. Ponder's built-in `/ready` flips as soon as historical sync reaches the tip. `/readyz` waits for all three conditions: Ponder synced, no chain lagging the tip, and `COUNT(historyBackfilled = false) = 0`. Expect it to report pending during the drain, which begins after `/ready` flips. Ponder reserves the `/ready` path, so `/readyz` is a distinct endpoint served by the app. -The staleness check reads each chain's newest indexed block from Ponder's `/status` and compares its timestamp against wall-clock time. Anything older than `READINESS_MAX_LAG_SECONDS` (default 300, overridable per chain with a numeric chain-id suffix) makes `/readyz` return 503 naming the chain, its newest block, and the measured lag. It deliberately does not call the RPC for the current head, because it wants to be robust to RPC outages. `/status` decodes the `_ponder_checkpoint` table, so the check reads committed database state rather than the indexing process's memory — which is what keeps it correct if the API is ever run separately from the indexer. +The staleness check reads each chain's newest indexed block from Ponder's `/status` and compares its timestamp against wall-clock time. Anything older than `READINESS_MAX_LAG_SECONDS` (default 300) makes `/readyz` return 503 naming the chain, its newest block, and the measured lag. It deliberately does not call the RPC for the current head, because it wants to be robust to RPC outages. Map these to different K8s probe types. The specific timing values (`periodSeconds`, `failureThreshold`, `initialDelaySeconds`) depend on your cluster's SLOs; what matters is which path and port to use: @@ -136,26 +136,22 @@ The 24-hour start period exists because a cold start legitimately fails `/readyz ### Splitting the API from the Indexer -Today one container does both jobs: `pnpm start` runs `ponder start`, which indexes and serves HTTP in a single process. Every indexer restart therefore takes the API down with it — including the automatic ones autoheal performs when the sync stalls. Ponder supports separating the two. +Today one container does both jobs: `pnpm start` runs `ponder start`, which indexes and serves HTTP in a single process. Every indexer restart therefore takes the API down with it. Ponder supports separating the two. This also useful for horizontal scaling the API side of the application. -`ponder serve` starts the HTTP server without the indexer. Same image, different command: one container keeps running `ponder start` (indexing, and incidentally still serving on its own port), a second runs `ponder serve` against the same database and schema, and the public route points at the second one. Restarting the indexer then leaves API traffic untouched. +`ponder serve` starts the HTTP server without the indexer. Same image, different command: one container keeps running `ponder start`, a second runs `ponder serve` against the same database and schema, and the public route points at the second one. Four constraints, read off the installed Ponder 0.16.6: | Aspect | Behaviour under `ponder serve` | |--------|-------------------------------| -| Database | Postgres only — it exits with an error on PGlite. | -| Schema | Must already exist. It logs `Schema does not exist` and exits 1, so the API container cannot start before the indexer has created the schema. Plan the startup ordering, or let it restart until the schema appears. | -| `/ready` | Works normally. It reads the `is_ready` flag from the database rather than from process state. | -| `/status` | Works normally. It decodes the `_ponder_checkpoint` table, so it reports the newest indexed block per chain regardless of which process is asking. | -| `ponder_sync_*` metrics | Absent. They are in-memory gauges set only by the indexing runtime (`runtime/realtime.js`, `runtime/historical.js`), which serve mode never runs. Nothing in this repo depends on them; `/api/sync-progress` reads the `ponder_historical_*` gauges and would report empty on an API-only container. | +| Database | Postgres only | +| Schema | Must already exist. | +| `/ready` | Works normally. | +| `/status` | Works normally. | +| `ponder_sync_*` metrics | Absent. They are in-memory gauges set only by the indexing runtime (`runtime/realtime.js`, `runtime/historical.js`), which serve mode never runs. | The RPC diagnostic is skipped in serve mode, but `ponder.config.ts` still executes, so give both containers the same environment rather than trimming the RPC variables from the API one. -`/readyz` works in either topology. Both conditions it evaluates — Ponder's `/ready` and the per-chain freshness check via `/status` — read committed database state, so an API-only container reports on the indexer's progress correctly rather than on its own idle process. This is the reason the freshness gate reads `/status` and not `ponder_sync_block_timestamp` from `/metrics`: the metrics route would have pinned the probe to the indexing process and made an API-only container fail readiness permanently. - -Be clear about what this does and does not buy. It keeps the API available across indexer restarts. It does nothing about staleness: during a stall the API stays up and serves stale data, which is the failure mode described under `/ready` vs `/readyz` Semantics. Splitting decouples availability from restarts; the readiness gate is what tells you the data has gone cold. - ### Structured Logging `pnpm start` runs with `--log-format json`, which makes both Ponder's internal log lines and the handler log lines emit newline-delimited JSON. Each handler log line includes structured fields (e.g. `chainId`, `block`) enabling log aggregators (Datadog, CloudWatch, Loki) to filter and alert by chain. @@ -220,9 +216,9 @@ A reindex that reuses an existing `ponder_sync` cache (same chain, same start bl `GET /ready` (Ponder built-in) returns `200` when Ponder has processed all historical blocks up to the tip and the live indexer is running. It does **not** guarantee historical discrete-order data is complete — `OwnerBackfillLive` drains that across live blocks after the tip. -`GET /readyz` (app) returns `200` only when Ponder is synced, no active chain has fallen behind the tip, **and** the owner backfill has finished (no non-deterministic historical generator with `historyBackfilled = false`). This is the promotion gate: it guarantees a newly-promoted pod has the full historical discrete-order set, so blue-green promotion never drops a complete pod for one that's still filling. It returns `503` (with the pending count) while the drain is in progress. +`GET /readyz` (app) returns `200` only when Ponder is synced, no active chain has fallen behind the tip, **and** the owner backfill has finished (no non-deterministic historical generator with `historyBackfilled = false`). It guarantees a newly-promoted pod has the full historical discrete-order set. It returns `503` (with the pending count) while the drain is in progress. -The staleness condition also makes `/readyz` useful after startup, which `/ready` is not. `/ready` latches at 200 once historical sync completes and never goes back, so an indexer whose realtime subscription dies keeps advertising itself as ready while its data silently ages. That happened in production: a dRPC WebSocket stopped delivering `newHeads` without closing the connection, Ponder logged one "No new block received within expected time" warning and then sat idle, and `/ready` plus the container health check both stayed green for a week. +The staleness condition also makes `/readyz` useful after startup, which `/ready` has a bug. During our tests, we notice that `/ready` latches at 200 once historical sync completes and doesn't go back in case of RPC websocket connection stopped delivering new blocks. During backfill both return `503`. GraphQL queries are still available but data is incomplete (generators and transactions accumulate; discrete orders fill in as live sync progresses).