feat: aggregated view for all twap parts

.github/workflows/ci.yml

@@ -38,3 +38,6 @@ jobs:
 
       - name: Test
         run: pnpm test
+
+      - name: Integration tests
+        run: pnpm test:int

AGENTS.md

@@ -24,6 +24,7 @@ ComposableCoW contract (per active chain — see src/chains/index.ts)
 - `abis/` — Contract ABIs
 - `src/chains/` — Chain configs and contract addresses (add a chain file, then register it in `src/chains/index.ts`)
 - `schema/tables.ts` — Table definitions; `schema/relations.ts` — Drizzle relations
+- `schema/views.ts` — Unified `partOrders` and `programmaticOrders` views; see `docs/api-reference.md` for query and sync guidance
 - `src/application/handlers/` — Event handlers (add new handlers here)
 - `src/api/index.ts` — Hono API exposing GraphQL and Ponder SQL client
 

docs/api-reference.md

@@ -22,10 +22,12 @@ The default local URL is `http://localhost:42069` when using `pnpm dev`. The pro
 
 ## GraphQL
 
-Ponder auto-generates the GraphQL schema from the tables in `ponder.schema.ts`. Open `/graphql` (or `/`) in a browser for GraphiQL — every table, field, and query argument is documented inline.
+Ponder auto-generates the GraphQL schema from the tables and views in `ponder.schema.ts`. Open `/graphql` (or `/`) in a browser for GraphiQL — every table, field, and query argument is documented inline.
 
 High-level map of what's queryable:
 
+- **`partOrder` / `partOrders`** — unified view of discrete orders and unconfirmed candidates. Each `(chainId, orderUid)` appears once, with the discrete row taking precedence.
+- **`programmaticOrder` / `programmaticOrders`** — generator view with `creationDate` and `partOrdersCount`. The count includes all unique known parts, including candidates, but not undiscovered parts.
 - **`conditionalOrderGenerator`** — one row per programmatic order registered via `ComposableCoW.create()` or `createWithContext()`. Holds decoded params, order type, `owner` (raw on-chain address), `resolvedOwner` (looked up in `ownerMapping` at insert time; falls back to `owner` if no mapping exists yet), and lifecycle status.
 - **`discreteOrder`** — individual CoW Protocol orders produced by a generator (a TWAP with 10 parts produces 10 discrete orders). Tracks orderbook status and executed amounts.
 - **`candidateDiscreteOrder`** — unconfirmed discrete orders discovered by the block handler, awaiting confirmation against the orderbook API.
@@ -35,6 +37,18 @@ High-level map of what's queryable:
 
 For schema details (columns, indexes, relations), see [architecture.md](./architecture.md).
 
+### Known parts and incremental sync
+
+Candidates have status `unconfirmed` and null execution amounts, regardless of parent status.
+`sortKey` orders parts by expiration, UID, and chain, with deterministic tie-breakers.
+
+1. Fetch all parents for the chain, filtered by `owner` OR `resolvedOwner`.
+2. Poll with inclusive `updatedAtBlock_gte`, using one cursor per chain and the same owner filter.
+3. Refetch all `partOrders` pages for changed parents, ordered by `sortKey`.
+4. Merge parents by `(chainId, eventId)` and parts by `(chainId, orderUid)`.
+
+New candidates and child order changes update the parent cursor. Parts have no separate cursor.
+
 ## REST endpoints
 
 Custom endpoints mounted at `/api`, documented in Swagger UI at `/docs`:

package.json

@@ -4,13 +4,15 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "dev": "ponder dev",
+    "dev": "pnpm run dev:db && ponder dev --hostname 127.0.0.1",
+    "dev:db": "POSTGRES_PORT=127.0.0.1:5432 docker compose up -d --wait postgres",
     "start": "ponder start -p 3000 --schema ${DATABASE_SCHEMA:-public} --log-format json",
     "db": "ponder db",
     "codegen": "ponder codegen",
     "lint": "eslint . --ext .ts",
     "typecheck": "tsc",
-    "test": "vitest run"
+    "test": "vitest run",
+    "test:int": "vitest run --config vitest.schema.config.ts"
   },
   "dependencies": {
     "@cowprotocol/cow-sdk": "^9.2.6",
@@ -25,6 +27,7 @@
     "zod": "^3.25.76"
   },
   "devDependencies": {
+    "@electric-sql/pglite": "0.2.13",
     "@types/node": "^20.9.0",
     "eslint": "^8.53.0",
     "eslint-config-ponder": "^0.16.2",

ponder.schema.ts

@@ -1,2 +1,3 @@
 export * from "./schema/tables";
 export * from "./schema/relations";
+export * from "./schema/views";

schema/tables.ts

@@ -114,8 +114,8 @@ export const conditionalOrderGenerator = onchainTable(
     consecutiveTryNextBlock: t.integer().notNull().default(0),  // Backoff counter for stuck generators
     historyBackfilled: t.boolean().notNull().default(false),    // OwnerBackfill has drained this generator's full /account history
     // Sync cursor: the indexer's processing block of the last client-relevant
-    // change (insert, status change, or any change to a child discrete order).
-    // NOT bumped for polling metadata or standalone allCandidatesKnown flips.
+    // change (insert, status change, or any change to a child part/candidate).
+    // NOT bumped for polling metadata alone.
     updatedAtBlock: t.bigint().notNull(),
     additionalData: t.json().$type<GeneratorAdditionalData>(),  // per-order-type extras; null unless the type defines any (only TWAP today)
   }),

schema/views.ts

@@ -0,0 +1,70 @@
+import { bigint, hex, onchainView, sql } from "ponder";
+import { integer, jsonb, text } from "drizzle-orm/pg-core";
+import {
+  candidateDiscreteOrder,
+  conditionalOrderGenerator,
+  discreteOrder,
+  orderStatusEnum,
+  orderTypeEnum,
+  transaction,
+} from "./tables";
+
+// Explicit column types keep computed fields filterable in Ponder GraphQL.
+export const partOrder = onchainView("part_order", {
+  orderUid: text("order_uid").notNull(),
+  chainId: integer("chain_id").notNull(),
+  conditionalOrderGeneratorId: text("conditional_order_generator_id").notNull(),
+  status: text("status").notNull(),
+  sellAmount: text("sell_amount").notNull(),
+  buyAmount: text("buy_amount").notNull(),
+  feeAmount: text("fee_amount").notNull(),
+  validTo: integer("valid_to"),
+  creationDate: bigint("creation_date").notNull(),
+  executedSellAmount: bigint("executed_sell_amount"),
+  executedBuyAmount: bigint("executed_buy_amount"),
+  executedFee: bigint("executed_fee"),
+  // Schedule order with a unique tie-breaker, unchanged by candidate promotion.
+  sortKey: text("sort_key").notNull(),
+}).as(sql`
+  select order_uid, chain_id, conditional_order_generator_id, status::text,
+    sell_amount, buy_amount, fee_amount, valid_to, creation_date,
+    executed_sell_amount, executed_buy_amount, executed_fee,
+    lpad(coalesce(valid_to, 0)::text, 10, '0') || ':' || order_uid || ':' || chain_id as sort_key
+  from ${discreteOrder}
+  union all
+  select c.order_uid, c.chain_id, c.conditional_order_generator_id, 'unconfirmed'::text,
+    c.sell_amount, c.buy_amount, c.fee_amount, c.valid_to, c.creation_date,
+    null::numeric, null::numeric, null::numeric,
+    lpad(coalesce(c.valid_to, 0)::text, 10, '0') || ':' || c.order_uid || ':' || c.chain_id
+  from ${candidateDiscreteOrder} c
+  left join ${discreteOrder} d on d.chain_id = c.chain_id and d.order_uid = c.order_uid
+  where d.order_uid is null
+`);
+
+// Views cannot be Drizzle relation targets. Expose the parent count here so it
+// uses the same deduplicated collection as the paginated parts endpoint.
+export const programmaticOrder = onchainView("programmatic_order", {
+  eventId: text("event_id").notNull(),
+  chainId: integer("chain_id").notNull(),
+  hash: hex("hash").notNull(),
+  owner: hex("owner").notNull(),
+  resolvedOwner: hex("resolved_owner"),
+  orderType: orderTypeEnum("order_type").notNull(),
+  status: orderStatusEnum("order_status").notNull(),
+  updatedAtBlock: bigint("updated_at_block").notNull(),
+  additionalData: jsonb("additional_data"),
+  decodedParams: jsonb("decoded_params"),
+  creationDate: bigint("creation_date").notNull(),
+  partOrdersCount: integer("part_orders_count").notNull(),
+}).as(sql`
+  select g.event_id, g.chain_id, g.hash, g.owner, g.resolved_owner, g.order_type,
+    g.order_status, g.updated_at_block, g.additional_data, g.decoded_params,
+    tx.block_timestamp as creation_date,
+    parts.part_orders_count
+  from ${conditionalOrderGenerator} g
+  inner join ${transaction} tx on tx.chain_id = g.chain_id and tx.hash = g.tx_hash
+  cross join lateral (
+    select count(*)::integer as part_orders_count from ${partOrder} p
+    where p.chain_id = g.chain_id and p.conditional_order_generator_id = g.event_id
+  ) parts
+`);

src/api/gql-docs/conditional-order-generator.ts

@@ -4,7 +4,7 @@ import {
   generateQueryDocs,
 } from "ponder-enrich-gql-docs-middleware";
 
-export const conditionalOrderGeneratorDocs: DocMap = {
+export const conditionalOrderGeneratorDocs = {
   conditionalOrderGenerator:
     "A programmatic order registered on-chain via ComposableCoW.create() or createWithContext(). One row per ConditionalOrderCreated event. Each generator may produce multiple discrete orders over its lifetime.",
   "conditionalOrderGenerator.eventId":
@@ -48,7 +48,7 @@ export const conditionalOrderGeneratorDocs: DocMap = {
   "conditionalOrderGenerator.historyBackfilled":
     "Whether OwnerBackfill has drained this generator's full /account order history from the CoW Orderbook. Applies to non-deterministic types (PerpetualSwap, GoodAfterTime, etc.) whose discrete orders cannot be precomputed. Internal one-time bootstrap flag.",
   "conditionalOrderGenerator.updatedAtBlock":
-    "Incremental-sync cursor: the indexer's processing block of the last client-relevant change — insert, status change, or any change to a child discrete order. NOT the block the change happened on-chain, and NOT bumped for internal polling fields (nextCheckBlock, lastCheckBlock, lastPollResult, consecutiveTryNextBlock, historyBackfilled) or standalone allCandidatesKnown flips, so treat those fields as potentially stale in a cursor-synced cache. To sync: fetch full history once, keep one cursor per chainId, then poll with updatedAtBlock_gte (inclusive — merge rows by (chainId, eventId), duplicates are idempotent), filtering by owner OR resolvedOwner like /orders-by-owner does. Then fetch changed parts via discreteOrder with conditionalOrderGeneratorId_in over the changed generators. Two known gaps, accepted by design: block reorgs may re-apply changes below your cursor (rows heal in the DB but a synced cache can miss them), and Aave-adapter generators created before their owner mapping was discovered keep resolvedOwner = adapter and are only picked up by an owner-filtered cursor query after their next status change.",
+    "Incremental-sync cursor: the indexer's processing block of the last client-relevant change, not the original on-chain event block. Generator inserts, status changes, child discrete-order changes, and new candidates update this cursor. UID precomputation also updates it when it sets allCandidatesKnown. Internal polling fields do not independently update it. Fetch full history once. Keep one cursor per chainId. Poll with updatedAtBlock_gte (inclusive), filtered by owner OR resolvedOwner. Merge generators by (chainId, eventId). Fetch all partOrders pages for changed generators with chainId and conditionalOrderGeneratorId_in. Merge parts by (chainId, orderUid). partOrders includes candidates but has no per-part update cursor. Reorgs can restore database rows at blocks earlier than the client cursor, so a cached view can miss changes. Older Aave-adapter generators can retain the adapter as resolvedOwner. An owner-filtered sync only receives them after their next cursor update.",
   "conditionalOrderGenerator.additionalData":
     "Per-order-type extra data (JSON). Only TWAP populates it today: { executedSellAmount, executedBuyAmount, executedFee } — totals aggregated across the generator's discrete orders, decimal strings in raw token units (fee in the sell token). Null for other order types.",
   "conditionalOrderGenerator.transaction":
@@ -60,4 +60,4 @@ export const conditionalOrderGeneratorDocs: DocMap = {
 
   ...generatePageDocs("conditionalOrderGenerator", "conditional order generator"),
   ...generateQueryDocs("conditionalOrderGenerator", "conditional order generator"),
-};
+} satisfies DocMap;

src/api/gql-docs/index.ts

@@ -8,13 +8,15 @@ import { discreteOrderDocs } from "./discrete-order";
 import { transactionDocs } from "./transaction";
 import { ownerMappingDocs } from "./owner-mapping";
 import { flashLoanOrderDocs } from "./flash-loan-order";
+import { viewDocs } from "./views";
 
 const docs = extendWithBaseDefinitions({
   ...conditionalOrderGeneratorDocs,
   ...discreteOrderDocs,
   ...transactionDocs,
   ...ownerMappingDocs,
   ...flashLoanOrderDocs,
+  ...viewDocs,
 });
 
 const _docsMiddleware = createDocumentationMiddleware(docs);

src/api/gql-docs/views.ts

@@ -0,0 +1,48 @@
+import {
+  DocMap,
+  generatePageDocs,
+  generateQueryDocs,
+} from "ponder-enrich-gql-docs-middleware";
+import { conditionalOrderGeneratorDocs } from "./conditional-order-generator";
+
+export const viewDocs: DocMap = {
+  partOrder:
+    "A known part from discreteOrder or candidateDiscreteOrder. For each (chainId, orderUid), the discrete row takes precedence. The view has no separate storage.",
+  "partOrder.orderUid": "CoW Protocol order UID. Together with chainId, identifies a unique part.",
+  "partOrder.chainId": "EVM chain ID.",
+  "partOrder.conditionalOrderGeneratorId": "The parent generator's eventId.",
+  "partOrder.status":
+    "open, fulfilled, unfilled, expired, cancelled, or unconfirmed. unconfirmed means a known candidate without a discrete row, including future scheduled parts. It does not guarantee that the part is currently executable. Other values come from discreteOrder.status.",
+  "partOrder.sellAmount": "Requested sell amount as a decimal string in raw token units.",
+  "partOrder.buyAmount": "Minimum buy amount as a decimal string in raw token units.",
+  "partOrder.feeAmount": "Fee amount as a decimal string in raw token units.",
+  "partOrder.validTo": "Expiration time in Unix seconds (UTC), as a JSON number. Null if unknown.",
+  "partOrder.creationDate":
+    "Observation time in Unix seconds (UTC), as a decimal string. Precomputed parts use the parent event timestamp. Use sortKey for deterministic pagination.",
+  "partOrder.executedSellAmount": "Executed sell amount in raw token units, as a decimal string. Null for candidates or unavailable execution data.",
+  "partOrder.executedBuyAmount": "Executed buy amount in raw token units, as a decimal string. Null for candidates or unavailable execution data.",
+  "partOrder.executedFee": "Executed fee in raw token units, as a decimal string. Null for candidates or unavailable execution data.",
+  "partOrder.sortKey":
+    "Deterministic sort key: validTo padded to ten digits (zero if null), then orderUid and chainId, separated by colons. UID and chain break expiration ties. The key stays unchanged during candidate promotion when these values stay unchanged. Use orderBy: sortKey with an explicit orderDirection.",
+
+  programmaticOrder:
+    "A generator view with its creation timestamp and the count of all known parts. It shares the generator's status and sync cursor. The view has no separate storage.",
+  "programmaticOrder.eventId": conditionalOrderGeneratorDocs["conditionalOrderGenerator.eventId"],
+  "programmaticOrder.chainId": conditionalOrderGeneratorDocs["conditionalOrderGenerator.chainId"],
+  "programmaticOrder.hash": conditionalOrderGeneratorDocs["conditionalOrderGenerator.hash"],
+  "programmaticOrder.owner": conditionalOrderGeneratorDocs["conditionalOrderGenerator.owner"],
+  "programmaticOrder.resolvedOwner": conditionalOrderGeneratorDocs["conditionalOrderGenerator.resolvedOwner"],
+  "programmaticOrder.orderType": conditionalOrderGeneratorDocs["conditionalOrderGenerator.orderType"],
+  "programmaticOrder.status": conditionalOrderGeneratorDocs["conditionalOrderGenerator.status"],
+  "programmaticOrder.updatedAtBlock": conditionalOrderGeneratorDocs["conditionalOrderGenerator.updatedAtBlock"],
+  "programmaticOrder.additionalData": conditionalOrderGeneratorDocs["conditionalOrderGenerator.additionalData"],
+  "programmaticOrder.decodedParams": conditionalOrderGeneratorDocs["conditionalOrderGenerator.decodedParams"],
+  "programmaticOrder.creationDate": "Parent transaction block timestamp in Unix seconds (UTC), as a decimal string.",
+  "programmaticOrder.partOrdersCount":
+    "Count of unique known parts for this generator and chain in partOrders, including unconfirmed candidates. Zero before discovery. Not necessarily the configured TWAP part count.",
+
+  ...generatePageDocs("partOrder", "known part order"),
+  ...generateQueryDocs("partOrder", "known part order"),
+  ...generatePageDocs("programmaticOrder", "programmatic order"),
+  ...generateQueryDocs("programmaticOrder", "programmatic order"),
+};

src/application/handlers/block/orderDiscoveryPoller.ts

@@ -24,6 +24,7 @@ import {
 } from "../../helpers/pollResultErrors";
 import { computeOrderUid, type GPv2OrderData } from "../../helpers/orderUid";
 import { log } from "../../helpers/logger";
+import { bumpGeneratorsUpdatedAt } from "../../helpers/updatedAtBlock";
 import { type OrderType } from "../../../utils/order-types";
 
 const SINGLE_SHOT_NON_DETERMINISTIC: readonly OrderType[] = ["GoodAfterTime", "TradeAboveThreshold"];
@@ -159,7 +160,11 @@ ponder.on("OrderDiscoveryPoller:block", async ({ event, context }) => {
             validTo: orderData.validTo,
             creationDate: event.block.timestamp,
           })
-          .onConflictDoNothing(),
+          .onConflictDoNothing()
+          .returning({ generatorId: candidateDiscreteOrder.conditionalOrderGeneratorId })
+          .then((inserted) => bumpGeneratorsUpdatedAt(
+            context, chainId, inserted.map((row) => row.generatorId), currentBlock,
+          )),
       );
 
       const isSingleShot = SINGLE_SHOT_NON_DETERMINISTIC.includes(order.orderType);

src/application/helpers/uidPrecompute.ts

@@ -214,7 +214,7 @@ export async function precomputeAndDiscover(
   // OrderDiscoveryPoller can skip this generator, OrderStatusTracker tracks the open orders.
   await context.db.sql
     .update(conditionalOrderGenerator)
-    .set({ allCandidatesKnown: true })
+    .set({ allCandidatesKnown: true, updatedAtBlock: blockNumber })
     .where(
       and(
         eq(conditionalOrderGenerator.chainId, chainId),