From c88165bbc677b7deff5de1175b1cd3a3c57b303c Mon Sep 17 00:00:00 2001 From: Aryan Godara Date: Thu, 13 Aug 2026 11:41:40 +0530 Subject: [PATCH 1/7] Add Balancer V2 indexer DB migrations --- database/README.md | 2 +- database/sql-pool-indexer/README.md | 37 ++++++++++++++++++- .../sql-pool-indexer/V111__balancer_v2.sql | 36 ++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 database/sql-pool-indexer/V111__balancer_v2.sql diff --git a/database/README.md b/database/README.md index 1e2e005818..df3c1a6db6 100644 --- a/database/README.md +++ b/database/README.md @@ -518,7 +518,7 @@ Indexes: - jit\_user\_order\_creation\_timestamp: btree(`owner`, `creation_timestamp` DESC) - jit\_event\_id: btree(`block_number`, `log_index`) -The `pool-indexer` service uses its own per-network database, not these shared DBs. Its tables (`pool_indexer_checkpoints`, `uniswap_v3_pools`, `uniswap_v3_pool_states`, `uniswap_v3_ticks`) and migrations live in [`sql-pool-indexer/`](sql-pool-indexer/). +The `pool-indexer` service uses its own per-network database, not these shared DBs. Its tables (`pool_indexer_checkpoints`, `uniswap_v3_pools`, `uniswap_v3_pool_states`, `uniswap_v3_ticks`, `balancer_v2_pools`, `balancer_v2_pool_tokens`) and migrations live in [`sql-pool-indexer/`](sql-pool-indexer/). ### Enums diff --git a/database/sql-pool-indexer/README.md b/database/sql-pool-indexer/README.md index fec9aa2f60..1af88317f2 100644 --- a/database/sql-pool-indexer/README.md +++ b/database/sql-pool-indexer/README.md @@ -18,11 +18,13 @@ applied migrations) so it's cancelled there by `../sql/V111`. ## Schema The tables below live in the indexer's own per-network database (e.g. -`ink_pool_indexer`), created by the migrations in this directory. +`ink_pool_indexer`), created by the migrations in this directory. `V110` defines +the Uniswap V3 discovery schema and `V111` the Balancer V2 one; a pool-indexer +process indexes whichever protocol(s) its network config enables. ### pool\_indexer\_checkpoints -Highest finalized block processed per `contract_address` by `pool-indexer`. `contract_address` is the factory address. The indexer runs one process per network against its own DB, so there's no `chain_id` column. +Highest finalized block processed per `contract_address` (the factory address) by `pool-indexer`. Shared by the Uniswap V3 and Balancer V2 indexers: their factory addresses are distinct contracts so rows never collide, and each protocol's queries filter to its own configured factories. One process per network against its own DB, so there's no `chain_id` column. Column | Type | Nullable | Details --------------------|--------|----------|-------- @@ -97,3 +99,34 @@ Quoters consult these to predict liquidity changes at tick crossings during swap Indexes: - PRIMARY KEY: btree (`pool_address`, `tick_idx`) + +### balancer\_v2\_pools + +One row per pool, discovered from each factory's `PoolCreated` event. `pool_type` is derived from the creating factory (weighted V0 and V3-plus both map to `Weighted`) and stored as the string the API serves. Referenced by `balancer_v2_pool_tokens`. Discovery metadata only — dynamic state (balances, amplification, LBP weights, scaling factors, swap fee) stays on-chain and is fetched by the driver at query time. + + Column | Type | Nullable | Details +----------------|--------|----------|-------- + pool\_id | bytea | not null | 32-byte Balancer poolId + address | bytea | not null | Pool address (poolId's first 20 bytes) + factory | bytea | not null | Factory that emitted `PoolCreated` + pool\_type | text | not null | `Weighted`, `Stable`, `ComposableStable`, or `LiquidityBootstrapping` (`CHECK`) + created\_block | bigint | not null | Block the pool was created on-chain + +Indexes: +- PRIMARY KEY: btree (`pool_id`) + +### balancer\_v2\_pool\_tokens + +Tokens per pool in `Vault.getPoolTokens` order. `decimals` is backfilled; `weight` is set only for weighted pools. FK → `balancer_v2_pools`. + + Column | Type | Nullable | Details +-----------|----------|----------|-------- + pool\_id | bytea | not null | FK → `balancer_v2_pools(pool_id)` + position | int | not null | Index in `getPoolTokens` order + token | bytea | not null | Token address + decimals | smallint | nullable | `NULL` = not yet fetched. `-1` = sentinel for "fetched but call failed" + weight | numeric | nullable | Bfp (1e18) normalized weight; weighted pools only, else `NULL` + +Indexes: +- PRIMARY KEY: btree (`pool_id`, `position`) +- Partial index on `(token)` with predicate `decimals IS NULL` to power the backfill scan. diff --git a/database/sql-pool-indexer/V111__balancer_v2.sql b/database/sql-pool-indexer/V111__balancer_v2.sql new file mode 100644 index 0000000000..98ce9a09c1 --- /dev/null +++ b/database/sql-pool-indexer/V111__balancer_v2.sql @@ -0,0 +1,36 @@ +-- Balancer V2 discovery tables, applied on top of V110's uniswap_v3_* schema in +-- the pool-indexer's per-network DB. Checkpoints reuse `pool_indexer_checkpoints` +-- (keyed by factory address): Balancer and Uniswap V3 factory addresses are +-- distinct contracts, so both protocols share that table without collision. + +-- One row per registered pool, discovered from each factory's `PoolCreated` +-- event. `pool_type` is derived from which factory created the pool (no on-chain +-- classification); weighted V0 and V3-plus both map to `Weighted` (the variant +-- is recoverable from `factory`). Stored as the string the API serves, so +-- there's no int<->enum mapping at the boundary. +CREATE TABLE balancer_v2_pools ( + pool_id BYTEA NOT NULL, -- 32-byte Balancer poolId + address BYTEA NOT NULL, -- pool address (poolId's first 20 bytes) + factory BYTEA NOT NULL, + pool_type TEXT NOT NULL CHECK (pool_type IN ('Weighted', 'Stable', 'ComposableStable', 'LiquidityBootstrapping')), + created_block BIGINT NOT NULL, + PRIMARY KEY (pool_id) +); + +-- Tokens per pool, in `Vault.getPoolTokens` order (`position`). `decimals` is +-- nullable and filled in by the backfill task. `weight` is the Balancer Bfp +-- (1e18 fixed-point) normalized weight, set only for weighted pools; NULL for +-- stable/composable-stable/LBP (their weights are absent or fetched on-chain). +CREATE TABLE balancer_v2_pool_tokens ( + pool_id BYTEA NOT NULL, + position INT NOT NULL, + token BYTEA NOT NULL, + decimals SMALLINT, -- NULL = not yet fetched; -1 = fetched but call failed + weight NUMERIC, -- Bfp (1e18); weighted pools only, else NULL + PRIMARY KEY (pool_id, position), + FOREIGN KEY (pool_id) REFERENCES balancer_v2_pools(pool_id) +); + +-- Decimals backfill hot path. Partial on `IS NULL` so the index shrinks to +-- near-empty once most rows are populated (real value or the `-1` sentinel). +CREATE INDEX ON balancer_v2_pool_tokens (token) WHERE decimals IS NULL; From 7ed43bf1acba4d7331dd4c158c5e2dfe39b5e2be Mon Sep 17 00:00:00 2001 From: Aryan Godara Date: Wed, 26 Aug 2026 20:42:33 +0530 Subject: [PATCH 2/7] Add ufixed18 to BigDecimal conversion for Balancer V2 weights --- crates/number/src/conversions.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/number/src/conversions.rs b/crates/number/src/conversions.rs index b63e87226e..985ff0c167 100644 --- a/crates/number/src/conversions.rs +++ b/crates/number/src/conversions.rs @@ -80,6 +80,13 @@ pub fn u256_to_big_decimal(u256: &U256) -> BigDecimal { BigDecimal::from(BigInt::from(big_uint)) } +/// Converts a `ufixed256x18` value (1e18 fixed-point — e.g. a Balancer `Bfp` +/// normalized weight) to its decimal value: `500000000000000000` -> +/// `0.500000000000000000`. +pub fn ufixed18_to_big_decimal(value: &U256) -> BigDecimal { + BigDecimal::new(u256_to_big_int(value), 18) +} + pub fn u160_to_big_decimal(u160: &U160) -> BigDecimal { let big_uint = BigUint::from_bytes_be(&u160.to_be_bytes::<20>()); BigDecimal::from(BigInt::from(big_uint)) From d9b096eb550c3a1a719f1e17d47de6f8b749adee Mon Sep 17 00:00:00 2001 From: Aryan Godara Date: Thu, 27 Aug 2026 13:14:48 +0530 Subject: [PATCH 3/7] Add Balancer V2 config to pool-indexer --- crates/e2e/tests/e2e/pool_indexer.rs | 20 +++-- crates/pool-indexer/src/config.rs | 126 ++++++++++++++++++++++++--- crates/pool-indexer/src/run.rs | 102 ++++++++++++---------- 3 files changed, 181 insertions(+), 67 deletions(-) diff --git a/crates/e2e/tests/e2e/pool_indexer.rs b/crates/e2e/tests/e2e/pool_indexer.rs index 73afccf9c9..990b3cd098 100644 --- a/crates/e2e/tests/e2e/pool_indexer.rs +++ b/crates/e2e/tests/e2e/pool_indexer.rs @@ -22,6 +22,7 @@ use { MetricsConfig, NetworkConfig, NetworkName, + UniswapV3Config, }, serde::Deserialize, sqlx::PgPool, @@ -204,18 +205,21 @@ fn pool_indexer_config( name: NetworkName::new("mainnet"), chain_id: 1, rpc_url: "http://127.0.0.1:8545".parse().unwrap(), - factories: factories - .into_iter() - .map(|address| FactoryConfig { - address, - deploy_block: 0, - }) - .collect(), - chunk_size: 1000, + uniswap_v3: Some(UniswapV3Config { + factories: factories + .into_iter() + .map(|address| FactoryConfig { + address, + deploy_block: 0, + }) + .collect(), + chunk_size: 1000, + }), poll_interval_secs: 1, use_latest: true, fetch_concurrency: 8, prefetch_concurrency: 50, + balancer_v2: None, }, api: ApiConfig { bind_address: SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, POOL_INDEXER_PORT)), diff --git a/crates/pool-indexer/src/config.rs b/crates/pool-indexer/src/config.rs index 629bb4876d..11ef3a34f5 100644 --- a/crates/pool-indexer/src/config.rs +++ b/crates/pool-indexer/src/config.rs @@ -3,6 +3,7 @@ use { anyhow::{Context, Result}, serde::Deserialize, std::{ + collections::HashSet, fmt, net::{Ipv4Addr, SocketAddr, SocketAddrV4}, num::NonZeroU32, @@ -16,10 +17,14 @@ const fn default_max_connections() -> NonZeroU32 { NonZeroU32::new(10).expect("non-zero literal") } -const fn default_chunk_size() -> u64 { +const fn default_uniswap_v3_chunk_size() -> u64 { 500 } +const fn default_balancer_v2_chunk_size() -> u64 { + 100_000 +} + const fn default_poll_interval_secs() -> u64 { 3 } @@ -82,26 +87,26 @@ pub struct NetworkConfig { pub chain_id: u64, #[serde(deserialize_with = "configs::deserialize_env::deserialize_url_from_env")] pub rpc_url: Url, - /// Uniswap V3 factories to index; non-empty and unique, enforced at parse. - #[serde(deserialize_with = "serde_ext::deserialize_nonempty_unique_vec")] - pub factories: Vec, - /// Blocks per `eth_getLogs` chunk during catch-up. - #[serde(default = "default_chunk_size")] - pub chunk_size: u64, /// Interval for polling for new blocks during live indexing. #[serde(default = "default_poll_interval_secs")] pub poll_interval_secs: u64, /// Number of `eth_getLogs` chunks fetched in parallel during live indexing. #[serde(default = "default_fetch_concurrency")] pub fetch_concurrency: usize, - /// `symbol()` / `decimals()` token-metadata RPC calls in flight during - /// the backfill passes. + /// Token-metadata RPC calls (`decimals()`, plus `symbol()` for Uniswap V3) + /// in flight during the backfill/enrich passes. #[serde(default = "default_prefetch_concurrency")] pub prefetch_concurrency: usize, /// Use `latest` instead of `finalized` as the indexing head. Set by tests /// against Anvil, which doesn't simulate finality. #[serde(skip)] pub use_latest: bool, + /// Uniswap V3 pools to index. Set on its own or alongside `balancer_v2`. + #[serde(default)] + pub uniswap_v3: Option, + /// Balancer V2 pools to index. Set on its own or alongside `uniswap_v3`. + #[serde(default)] + pub balancer_v2: Option, } impl NetworkConfig { @@ -109,29 +114,121 @@ impl NetworkConfig { Duration::from_secs(self.poll_interval_secs) } - pub fn indexer_config(&self, factory: Address) -> IndexerConfig { + pub fn indexer_config(&self, chunk_size: u64, factory: Address) -> IndexerConfig { IndexerConfig { network: self.name.clone(), chain_id: self.chain_id, factory_address: factory, - chunk_size: self.chunk_size, + chunk_size, use_latest: self.use_latest, fetch_concurrency: self.fetch_concurrency, prefetch_concurrency: self.prefetch_concurrency, } } + + /// Cross-field checks: index at least one protocol, and every factory + /// address (across both) is unique — checkpoints are keyed by factory + /// address in the shared `pool_indexer_checkpoints`, so a repeated address + /// would drive two indexer loops onto one row. + fn validate(&self) -> Result<()> { + anyhow::ensure!( + self.uniswap_v3.is_some() || self.balancer_v2.is_some(), + "network {}: configure at least one of `uniswap-v3` or `balancer-v2`", + self.name, + ); + if let Some(balancer) = &self.balancer_v2 { + anyhow::ensure!( + balancer.factory_count() > 0, + "network {}: balancer-v2 requires at least one factory", + self.name, + ); + } + let uniswap_factories = self + .uniswap_v3 + .iter() + .flat_map(|u| &u.factories) + .map(|f| f.address); + let balancer_factories = self + .balancer_v2 + .iter() + .flat_map(|b| b.factories()) + .map(|f| f.address); + let mut seen = HashSet::new(); + for factory in uniswap_factories.chain(balancer_factories) { + anyhow::ensure!( + seen.insert(factory), + "network {}: factory {factory} configured more than once", + self.name, + ); + } + Ok(()) + } } -/// The factory and the block it was deployed at. The indexer cold-seeds by -/// replaying on-chain events from `deploy_block`, then live-indexes. +/// A pool factory and the block it was deployed at; the indexer cold-seeds by +/// replaying `PoolCreated` from `deploy_block`, then live-indexes. Shared by +/// the Uniswap V3 and Balancer V2 configs. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub struct FactoryConfig { pub address: Address, - /// Block the factory was deployed at; on-chain cold-seed scans from here + /// Block the factory was deployed at; on-chain cold-seed scans from here. pub deploy_block: u64, } +/// Uniswap V3 discovery config: the factories whose `PoolCreated` events the +/// indexer scans. Non-empty and unique, enforced at parse. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct UniswapV3Config { + #[serde(deserialize_with = "serde_ext::deserialize_nonempty_unique_vec")] + pub factories: Vec, + /// Blocks per `eth_getLogs` chunk. Small: the indexer fetches every pool's + /// events chain-wide, so a wide range blows past RPC log-response caps. + #[serde(default = "default_uniswap_v3_chunk_size")] + pub chunk_size: u64, +} + +/// Balancer V2 indexer config. Pools are created by per-type factories and +/// registered with a single Vault; the pool type is implied by which group a +/// factory is listed under. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct BalancerV2Config { + pub vault: Address, + /// Blocks per `eth_getLogs` chunk. Large: `PoolCreated` is factory-filtered + /// (few logs), so the cold-seed scan stays a handful of calls. + #[serde(default = "default_balancer_v2_chunk_size")] + pub chunk_size: u64, + #[serde(default)] + pub weighted: Vec, + #[serde(default)] + pub weighted_v3plus: Vec, + #[serde(default)] + pub stable: Vec, + #[serde(default)] + pub liquidity_bootstrapping: Vec, + #[serde(default)] + pub composable_stable: Vec, +} + +impl BalancerV2Config { + /// All configured factories, across every pool type. + fn factories(&self) -> impl Iterator { + self.weighted + .iter() + .chain(&self.weighted_v3plus) + .chain(&self.stable) + .chain(&self.liquidity_bootstrapping) + .chain(&self.composable_stable) + } + + /// Total factories configured across all pool types. + pub fn factory_count(&self) -> usize { + self.factories().count() + } +} + /// Subset of [`NetworkConfig`] handed to [`UniswapV3Indexer`] at runtime. #[derive(Debug, Clone)] pub struct IndexerConfig { @@ -190,6 +287,7 @@ impl Configuration { let content = std::fs::read_to_string(path) .with_context(|| format!("reading config file {}", path.display()))?; let parsed: Self = toml::from_str(&content).context("parsing config file")?; + parsed.network.validate()?; Ok(parsed) } } diff --git a/crates/pool-indexer/src/run.rs b/crates/pool-indexer/src/run.rs index 71e173b071..44d85d59f8 100644 --- a/crates/pool-indexer/src/run.rs +++ b/crates/pool-indexer/src/run.rs @@ -47,17 +47,19 @@ pub async fn bootstrap(config: Configuration) { // Seed every factory concurrently, like the serve path. let mut factory_set = JoinSet::new(); - for factory in network.factories.iter().copied() { - let indexer = UniswapV3Indexer::new( - provider.clone(), - db.clone(), - &network.indexer_config(factory.address), - ); - let db = db.clone(); - let network = network.clone(); - factory_set.spawn(async move { - bootstrap_factory(&db, &indexer, &network, &factory).await; - }); + if let Some(uniswap_v3) = &network.uniswap_v3 { + for factory in uniswap_v3.factories.iter().copied() { + let indexer = UniswapV3Indexer::new( + provider.clone(), + db.clone(), + &network.indexer_config(uniswap_v3.chunk_size, factory.address), + ); + let db = db.clone(); + let network = network.clone(); + factory_set.spawn(async move { + bootstrap_factory(&db, &indexer, &network, &factory).await; + }); + } } while let Some(result) = factory_set.join_next().await { result.expect("bootstrap task panicked"); @@ -72,7 +74,11 @@ pub async fn run(config: Configuration) { let startup = Arc::new(Some(AtomicBool::new(false))); let barrier = Arc::new(StartupBarrier::new( startup.clone(), - config.network.factories.len(), + config + .network + .uniswap_v3 + .as_ref() + .map_or(0, |u| u.factories.len()), )); // Abort the metrics task when `run` exits, so tests can rebind the port. @@ -153,7 +159,12 @@ fn build_api_state(db: &PgPool, network: &NetworkConfig) -> Arc { Arc::new(AppState { db: db.clone(), network: network.name.clone(), - factories: network.factories.iter().map(|f| f.address).collect(), + factories: network + .uniswap_v3 + .iter() + .flat_map(|u| &u.factories) + .map(|f| f.address) + .collect(), }) } @@ -161,51 +172,52 @@ async fn run_network_indexer(db: PgPool, network: NetworkConfig, barrier: Arc Date: Thu, 27 Aug 2026 15:27:09 +0530 Subject: [PATCH 4/7] add bal-v2 discovery to pool-indexer --- crates/pool-indexer/src/db/balancer_v2.rs | 116 ++++ crates/pool-indexer/src/db/mod.rs | 45 ++ crates/pool-indexer/src/db/uniswap_v3.rs | 39 +- .../pool-indexer/src/indexer/balancer_v2.rs | 591 ++++++++++++++++++ crates/pool-indexer/src/indexer/mod.rs | 158 +++++ crates/pool-indexer/src/indexer/uniswap_v3.rs | 169 +---- crates/pool-indexer/src/run.rs | 85 ++- 7 files changed, 1001 insertions(+), 202 deletions(-) create mode 100644 crates/pool-indexer/src/db/balancer_v2.rs create mode 100644 crates/pool-indexer/src/indexer/balancer_v2.rs diff --git a/crates/pool-indexer/src/db/balancer_v2.rs b/crates/pool-indexer/src/db/balancer_v2.rs new file mode 100644 index 0000000000..c30674771d --- /dev/null +++ b/crates/pool-indexer/src/db/balancer_v2.rs @@ -0,0 +1,116 @@ +use { + crate::{db::bytes_to_addr, indexer::balancer_v2::NewBalancerPool}, + alloy_primitives::Address, + anyhow::{Context, Result}, + bigdecimal::BigDecimal, + sqlx::{PgPool, Postgres, Row, Transaction}, +}; + +/// Inserts discovered pools and their tokens. Pools are written before tokens +/// to satisfy the FK; both `ON CONFLICT DO NOTHING` so re-indexing a pool is a +/// no-op. +pub async fn insert_pools( + tx: &mut Transaction<'_, Postgres>, + factory: &Address, + pools: &[NewBalancerPool], +) -> Result<()> { + if pools.is_empty() { + return Ok(()); + } + + let mut pool_ids: Vec<&[u8]> = Vec::with_capacity(pools.len()); + let mut addresses: Vec<&[u8]> = Vec::with_capacity(pools.len()); + let mut pool_types: Vec<&str> = Vec::with_capacity(pools.len()); + let mut created_blocks: Vec = Vec::with_capacity(pools.len()); + let mut tok_pool_ids: Vec<&[u8]> = Vec::new(); + let mut positions: Vec = Vec::new(); + let mut tokens: Vec<&[u8]> = Vec::new(); + let mut decimals: Vec> = Vec::new(); + let mut weights: Vec> = Vec::new(); + for pool in pools { + pool_ids.push(pool.pool_id.as_slice()); + addresses.push(pool.address.as_slice()); + pool_types.push(pool.pool_type.as_str()); + created_blocks.push(pool.created_block.cast_signed()); + for token in &pool.tokens { + tok_pool_ids.push(pool.pool_id.as_slice()); + positions.push(i32::try_from(token.position).unwrap_or(i32::MAX)); + tokens.push(token.address.as_slice()); + decimals.push(token.decimals.map(i16::from)); + weights.push(token.weight.clone()); + } + } + + sqlx::query( + "INSERT INTO balancer_v2_pools (pool_id, address, factory, pool_type, created_block) + SELECT t.pid, t.addr, $1, t.ptype, t.cblk + FROM UNNEST($2::BYTEA[], $3::BYTEA[], $4::TEXT[], $5::INT8[]) + AS t(pid, addr, ptype, cblk) + ON CONFLICT (pool_id) DO NOTHING", + ) + .bind(factory.as_slice()) + .bind(pool_ids) + .bind(addresses) + .bind(pool_types) + .bind(created_blocks) + .execute(&mut **tx) + .await + .context("insert balancer pools")?; + + sqlx::query( + "INSERT INTO balancer_v2_pool_tokens (pool_id, position, token, decimals, weight) + SELECT t.pid, t.pos, t.tok, t.dec, t.wgt + FROM UNNEST($1::BYTEA[], $2::INT4[], $3::BYTEA[], $4::INT2[], $5::NUMERIC[]) + AS t(pid, pos, tok, dec, wgt) + ON CONFLICT (pool_id, position) DO NOTHING", + ) + .bind(tok_pool_ids) + .bind(positions) + .bind(tokens) + .bind(decimals) + .bind(weights) + .execute(&mut **tx) + .await + .context("insert balancer pool tokens")?; + + Ok(()) +} + +/// Distinct token addresses with no `decimals` recorded yet. +pub async fn get_tokens_missing_decimals(pool: &PgPool) -> Result> { + let rows = + sqlx::query("SELECT DISTINCT token FROM balancer_v2_pool_tokens WHERE decimals IS NULL") + .fetch_all(pool) + .await + .context("get_tokens_missing_decimals")?; + + rows.into_iter() + .map(|r| bytes_to_addr(r.get("token"))) + .collect() +} + +/// Sets `decimals` for every token row matching one of the inputs. Pass `-1` +/// for "tried, failed" so the next backfill's `IS NULL` filter still skips it. +pub async fn batch_set_token_decimals( + tx: &mut Transaction<'_, Postgres>, + entries: &[(Address, i16)], +) -> Result<()> { + if entries.is_empty() { + return Ok(()); + } + let tokens: Vec<&[u8]> = entries.iter().map(|(t, _)| t.as_slice()).collect(); + let decimals: Vec = entries.iter().map(|(_, d)| *d).collect(); + + sqlx::query( + "UPDATE balancer_v2_pool_tokens p + SET decimals = i.dec + FROM UNNEST($1::BYTEA[], $2::INT2[]) AS i(tok, dec) + WHERE p.token = i.tok AND p.decimals IS NULL", + ) + .bind(tokens) + .bind(decimals) + .execute(&mut **tx) + .await + .context("batch_set_token_decimals")?; + Ok(()) +} diff --git a/crates/pool-indexer/src/db/mod.rs b/crates/pool-indexer/src/db/mod.rs index c24144d0e0..433ed0f816 100644 --- a/crates/pool-indexer/src/db/mod.rs +++ b/crates/pool-indexer/src/db/mod.rs @@ -1 +1,46 @@ +pub mod balancer_v2; pub mod uniswap_v3; + +use { + alloy_primitives::Address, + anyhow::{Context, Result}, + sqlx::{PgPool, Postgres, Row, Transaction}, +}; + +/// Decodes a Postgres `BYTEA` column into an [`Address`]. +pub(crate) fn bytes_to_addr(b: Vec) -> Result
{ + Address::try_from(b.as_slice()).context("invalid address bytes") +} + +/// Highest block scanned for a factory's `PoolCreated` events. Shared by both +/// indexers: `pool_indexer_checkpoints` is keyed by factory address, which is +/// unique across protocols, so their rows never collide. +pub async fn get_checkpoint(pool: &PgPool, factory: &Address) -> Result> { + let row = sqlx::query( + "SELECT block_number FROM pool_indexer_checkpoints WHERE contract_address = $1", + ) + .bind(factory.as_slice()) + .fetch_optional(pool) + .await + .context("get_checkpoint")?; + + Ok(row.map(|r| r.get::("block_number").cast_unsigned())) +} + +pub async fn set_checkpoint( + tx: &mut Transaction<'_, Postgres>, + factory: &Address, + block_number: u64, +) -> Result<()> { + sqlx::query( + "INSERT INTO pool_indexer_checkpoints (contract_address, block_number) + VALUES ($1, $2) + ON CONFLICT (contract_address) DO UPDATE SET block_number = EXCLUDED.block_number", + ) + .bind(factory.as_slice()) + .bind(block_number.cast_signed()) + .execute(&mut **tx) + .await + .context("set_checkpoint")?; + Ok(()) +} diff --git a/crates/pool-indexer/src/db/uniswap_v3.rs b/crates/pool-indexer/src/db/uniswap_v3.rs index fa0273f662..cb195428e0 100644 --- a/crates/pool-indexer/src/db/uniswap_v3.rs +++ b/crates/pool-indexer/src/db/uniswap_v3.rs @@ -1,5 +1,8 @@ use { - crate::indexer::uniswap_v3::{LiquidityUpdateData, NewPoolData, PoolStateData, TickDeltaData}, + crate::{ + db::bytes_to_addr, + indexer::uniswap_v3::{LiquidityUpdateData, NewPoolData, PoolStateData, TickDeltaData}, + }, alloy_primitives::Address, anyhow::{Context, Result}, bigdecimal::BigDecimal, @@ -9,10 +12,6 @@ use { std::collections::BTreeSet, }; -fn bytes_to_addr(b: Vec) -> Result
{ - Address::try_from(b.as_slice()).context("invalid address bytes") -} - fn address_bytes_list(addresses: &[Address]) -> Vec<&[u8]> { addresses.iter().map(|address| address.as_slice()).collect() } @@ -21,36 +20,6 @@ fn decode_pool_rows(rows: Vec) -> Result> { rows.into_iter().map(PoolRow::try_from).collect() } -pub async fn get_checkpoint(pool: &PgPool, contract: &Address) -> Result> { - let row = sqlx::query( - "SELECT block_number FROM pool_indexer_checkpoints WHERE contract_address = $1", - ) - .bind(contract.as_slice()) - .fetch_optional(pool) - .await - .context("get_checkpoint")?; - - Ok(row.map(|r| r.get::("block_number").cast_unsigned())) -} - -pub async fn set_checkpoint( - tx: &mut Transaction<'_, Postgres>, - contract: &Address, - block_number: u64, -) -> Result<()> { - sqlx::query( - "INSERT INTO pool_indexer_checkpoints (contract_address, block_number) - VALUES ($1, $2) - ON CONFLICT (contract_address) DO UPDATE SET block_number = EXCLUDED.block_number", - ) - .bind(contract.as_slice()) - .bind(block_number.cast_signed()) - .execute(&mut **tx) - .await - .context("set_checkpoint")?; - Ok(()) -} - pub async fn insert_pools( tx: &mut Transaction<'_, Postgres>, factory: &Address, diff --git a/crates/pool-indexer/src/indexer/balancer_v2.rs b/crates/pool-indexer/src/indexer/balancer_v2.rs new file mode 100644 index 0000000000..1a794a0ab8 --- /dev/null +++ b/crates/pool-indexer/src/indexer/balancer_v2.rs @@ -0,0 +1,591 @@ +use { + crate::{ + config::{BalancerV2Config, FactoryConfig, NetworkName}, + db::{balancer_v2 as db, get_checkpoint, set_checkpoint}, + }, + alloy_primitives::{Address, B256, U256}, + alloy_provider::Provider, + alloy_rpc_types_eth::{BlockNumberOrTag, Log}, + alloy_sol_types::SolEvent, + anyhow::{Context, Result}, + bigdecimal::BigDecimal, + contracts::{ + BalancerV2BasePool, + BalancerV2BasePoolFactory::BalancerV2BasePoolFactory::PoolCreated, + BalancerV2Vault, + BalancerV2WeightedPool, + }, + ethrpc::AlloyProvider, + futures::{StreamExt, TryStreamExt}, + number::conversions::ufixed18_to_big_decimal, + sqlx::PgPool, + std::collections::HashMap, + tracing::instrument, +}; + +const BACKFILL_BATCH_SIZE: usize = 500; + +/// Balancer V2 pool type, implied by the factory group a pool was discovered +/// under. Stored as [`Self::as_str`] and served verbatim by the API; the +/// weighted V0/V3-plus distinction is recovered from the factory address, so +/// it isn't a separate type. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PoolType { + Weighted, + Stable, + ComposableStable, + LiquidityBootstrapping, +} + +impl PoolType { + /// String stored in `balancer_v2_pools.pool_type`; matches the schema's + /// `CHECK` constraint. + pub fn as_str(self) -> &'static str { + match self { + PoolType::Weighted => "Weighted", + PoolType::Stable => "Stable", + PoolType::ComposableStable => "ComposableStable", + PoolType::LiquidityBootstrapping => "LiquidityBootstrapping", + } + } + + /// Whether pools of this type expose `getNormalizedWeights`. Only weighted + /// pools carry static normalized weights; stable/composable-stable/LBP + /// weights are absent or computed on-chain, so they aren't fetched here. + pub fn has_weights(self) -> bool { + matches!(self, PoolType::Weighted) + } +} + +/// Flattens a [`BalancerV2Config`] into `(pool_type, factory)` pairs across all +/// factory groups. The pool type is implied by the group; both weighted groups +/// map to [`PoolType::Weighted`]. +pub fn configured_factories(config: &BalancerV2Config) -> Vec<(PoolType, FactoryConfig)> { + let groups: [(PoolType, &[FactoryConfig]); 5] = [ + (PoolType::Weighted, &config.weighted), + (PoolType::Weighted, &config.weighted_v3plus), + (PoolType::Stable, &config.stable), + ( + PoolType::LiquidityBootstrapping, + &config.liquidity_bootstrapping, + ), + (PoolType::ComposableStable, &config.composable_stable), + ]; + groups + .into_iter() + .flat_map(|(pool_type, factories)| factories.iter().map(move |f| (pool_type, *f))) + .collect() +} + +/// Config for one Balancer V2 factory discovery loop. +pub struct IndexerConfig { + pub network: NetworkName, + pub vault: Address, + pub factory: Address, + pub pool_type: PoolType, + pub deploy_block: u64, + pub chunk_size: u64, + pub use_latest: bool, + pub fetch_concurrency: usize, + pub enrich_concurrency: usize, +} + +/// A token within a discovered pool, in `Vault.getPoolTokens` order. +pub struct NewPoolToken { + pub position: usize, + pub address: Address, + pub decimals: Option, + /// Normalized weight as a Bfp (1e18) fraction; `Some` only for weighted + /// pools. + pub weight: Option, +} + +/// A pool discovered from a factory `PoolCreated` event, enriched with on-chain +/// metadata. +pub struct NewBalancerPool { + pub pool_id: B256, + pub address: Address, + pub pool_type: PoolType, + pub created_block: u64, + pub tokens: Vec, +} + +#[derive(Clone, Copy, Debug)] +struct ChunkRange { + start: u64, + end: u64, +} + +/// Discovers Balancer V2 pools created by one factory and persists their +/// static metadata. Dynamic state (balances, amp, swap fee, ...) is fetched +/// on-chain by the driver, not here. +pub struct BalancerV2Indexer { + provider: AlloyProvider, + db: PgPool, + network: NetworkName, + vault: Address, + factory: Address, + factory_label: String, + pool_type: PoolType, + deploy_block: u64, + chunk_size: u64, + finality_tag: BlockNumberOrTag, + fetch_concurrency: usize, + enrich_concurrency: usize, +} + +impl BalancerV2Indexer { + pub fn new(provider: AlloyProvider, db: PgPool, config: IndexerConfig) -> Self { + Self { + provider, + db, + network: config.network, + vault: config.vault, + factory: config.factory, + factory_label: format!("{:#x}", config.factory), + pool_type: config.pool_type, + deploy_block: config.deploy_block, + chunk_size: config.chunk_size, + finality_tag: if config.use_latest { + BlockNumberOrTag::Latest + } else { + BlockNumberOrTag::Finalized + }, + fetch_concurrency: config.fetch_concurrency, + enrich_concurrency: config.enrich_concurrency, + } + } + + /// Per-factory live-discovery loop. + pub async fn run(self, poll_interval: std::time::Duration) -> ! { + let mut interval = tokio::time::interval(poll_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + interval.tick().await; + if let Err(err) = self.run_once().await { + crate::metrics::Metrics::get() + .indexer_errors + .with_label_values(&[self.network.as_str(), self.factory_label.as_str()]) + .inc(); + tracing::error!(?err, factory = %self.factory, "balancer indexer error, retrying"); + } + } + } + + /// Scans `PoolCreated` from the factory's deploy block to the finalized + /// head, then returns. Idempotent: skips if a checkpoint already exists, so + /// re-running on a seeded DB is a fast no-op. + pub async fn bootstrap(&self) -> Result<()> { + if get_checkpoint(&self.db, &self.factory).await?.is_some() { + tracing::info!(factory = %self.factory, "existing checkpoint, skipping bootstrap"); + return Ok(()); + } + // `run_once` resumes at `checkpoint + 1`, so start one before the + // deploy block to scan the deploy block itself. + let start = self.deploy_block.saturating_sub(1); + let mut tx = self.db.begin().await.context("begin checkpoint tx")?; + set_checkpoint(&mut tx, &self.factory, start).await?; + tx.commit().await.context("commit checkpoint tx")?; + + loop { + let finalized = self.finalized_block().await?; + if self.last_indexed_block().await? >= finalized { + tracing::info!(block = finalized, factory = %self.factory, "balancer bootstrap caught up"); + return Ok(()); + } + self.run_once().await?; + } + } + + async fn run_once(&self) -> Result<()> { + let finalized = self.finalized_block().await?; + let last = self.last_indexed_block().await?; + let lag = finalized.saturating_sub(last); + crate::metrics::Metrics::get() + .indexer_lag_blocks + .with_label_values(&[self.network.as_str(), self.factory_label.as_str()]) + .set(i64::try_from(lag).unwrap_or(i64::MAX)); + if last >= finalized { + return Ok(()); + } + + // Fetch chunks in parallel, commit in order. + futures::stream::iter(self.pending_chunks(last, finalized)) + .map(|chunk| async move { + let logs = self.fetch_pool_created(chunk.start, chunk.end).await?; + Ok::<_, anyhow::Error>((chunk, logs)) + }) + .buffered(self.fetch_concurrency) + .try_for_each(|(chunk, logs)| self.commit_chunk(chunk, logs, finalized)) + .await?; + Ok(()) + } + + async fn finalized_block(&self) -> Result { + Ok(self + .provider + .get_block_by_number(self.finality_tag) + .await + .context("get finalized block")? + .context("no finalized block")? + .header + .number) + } + + async fn last_indexed_block(&self) -> Result { + Ok(get_checkpoint(&self.db, &self.factory).await?.unwrap_or(0)) + } + + fn pending_chunks(&self, last: u64, finalized: u64) -> Vec { + let mut chunks = Vec::new(); + let mut next_start = last + 1; + while next_start <= finalized { + let end = (next_start + self.chunk_size - 1).min(finalized); + chunks.push(ChunkRange { + start: next_start, + end, + }); + next_start = end + 1; + } + chunks + } + + /// `PoolCreated` is emitted by the factory only, so filter by the factory + /// address — the query stays tiny even over large ranges. + async fn fetch_pool_created(&self, from: u64, to: u64) -> Result> { + super::bisecting_get_logs( + &self.provider, + from, + to, + vec![self.factory], + vec![PoolCreated::SIGNATURE_HASH], + ) + .await + } + + #[instrument(skip(self, logs), fields(chunk_start = chunk.start, chunk_end = chunk.end))] + async fn commit_chunk(&self, chunk: ChunkRange, logs: Vec, target: u64) -> Result<()> { + let pools: Vec = + futures::stream::iter(pool_addresses(self.factory, &logs)) + .map(|(pool, block)| self.enrich_pool(pool, block)) + .buffer_unordered(self.enrich_concurrency) + .filter_map(|res| async move { + match res { + Ok(pool) => pool, + Err(err) => { + tracing::warn!(?err, "balancer pool enrichment failed; skipping"); + None + } + } + }) + .collect() + .await; + + let network = self.network.as_str(); + let factory = self.factory_label.as_str(); + crate::metrics::Metrics::get() + .events_applied + .with_label_values(&[network, factory, "new_pool"]) + .inc_by(pools.len() as u64); + + let mut tx = self.db.begin().await.context("begin transaction")?; + db::insert_pools(&mut tx, &self.factory, &pools).await?; + set_checkpoint(&mut tx, &self.factory, chunk.end).await?; + tx.commit().await.context("commit transaction")?; + + let metrics = crate::metrics::Metrics::get(); + metrics + .indexed_block + .with_label_values(&[network, factory]) + .set(i64::try_from(chunk.end).unwrap_or(i64::MAX)); + let lag = target.saturating_sub(chunk.end); + metrics + .indexer_lag_blocks + .with_label_values(&[network, factory]) + .set(i64::try_from(lag).unwrap_or(i64::MAX)); + Ok(()) + } + + /// Enriches a freshly-discovered pool with its on-chain metadata: + /// `getPoolId` → `Vault.getPoolTokens` → per-token `decimals` → + /// `getNormalizedWeights` (weighted pools). Returns `None` if a required + /// call fails (the pool is retried on the next pass — the checkpoint isn't + /// advanced past it until it inserts). + async fn enrich_pool( + &self, + pool: Address, + created_block: u64, + ) -> Result> { + let pool_id = match BalancerV2BasePool::Instance::new(pool, self.provider.clone()) + .getPoolId() + .call() + .await + { + Ok(id) => id, + Err(err) => { + tracing::warn!(%pool, ?err, "getPoolId failed; skipping pool"); + return Ok(None); + } + }; + + let tokens = BalancerV2Vault::Instance::new(self.vault, self.provider.clone()) + .getPoolTokens(pool_id.0.into()) + .call() + .await + .context("getPoolTokens")? + .tokens; + + let weights = if self.pool_type.has_weights() { + match BalancerV2WeightedPool::Instance::new(pool, self.provider.clone()) + .getNormalizedWeights() + .call() + .await + { + Ok(weights) => Some(weights), + Err(err) => { + tracing::warn!(%pool, ?err, "getNormalizedWeights failed; skipping pool"); + return Ok(None); + } + } + } else { + None + }; + + let decimals: HashMap = futures::stream::iter(tokens.clone()) + .map(|token| async move { (token, super::fetch_decimals(&self.provider, token).await) }) + .buffer_unordered(self.enrich_concurrency) + .filter_map(|(token, decimals)| async move { decimals.map(|d| (token, d)) }) + .collect() + .await; + + Ok(Some(assemble_pool( + pool, + pool_id, + self.pool_type, + created_block, + tokens, + &decimals, + weights, + ))) + } +} + +/// Decodes factory `PoolCreated` events into `(pool address, created block)`. +/// Logs are already factory-filtered; the emitter check is defense-in-depth. +fn pool_addresses(factory: Address, logs: &[Log]) -> Vec<(Address, u64)> { + logs.iter() + .filter_map(|log| { + let topic = log.topic0()?; + if *topic != PoolCreated::SIGNATURE_HASH || log.address() != factory { + return None; + } + let decoded = PoolCreated::decode_log(&log.inner).ok()?; + Some((decoded.data.pool, log.block_number.unwrap_or_default())) + }) + .collect() +} + +/// Builds a pool row from its enrichment results. Tokens keep their +/// `getPoolTokens` order via `position`; `weights[i]` aligns with `tokens[i]` +/// and is converted from 1e18 fixed-point to a decimal fraction. Pure (no I/O) +/// so the mapping is unit-testable. +fn assemble_pool( + pool: Address, + pool_id: B256, + pool_type: PoolType, + created_block: u64, + tokens: Vec
, + decimals: &HashMap, + weights: Option>, +) -> NewBalancerPool { + let weights = weights.unwrap_or_default(); + let tokens = tokens + .into_iter() + .enumerate() + .map(|(position, address)| NewPoolToken { + position, + address, + decimals: decimals.get(&address).copied(), + weight: weights.get(position).map(ufixed18_to_big_decimal), + }) + .collect(); + NewBalancerPool { + pool_id, + address: pool, + pool_type, + created_block, + tokens, + } +} + +/// Periodically fills `decimals` on pool tokens whose discovery-time +/// `decimals()` call failed (or was never made). `-1` is the "tried, failed" +/// sentinel so a known-broken token isn't probed every pass. +pub(crate) async fn backfill_decimals( + provider: AlloyProvider, + db: PgPool, + network: NetworkName, + concurrency: usize, + poll_interval: std::time::Duration, +) { + let mut interval = tokio::time::interval(poll_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + interval.tick().await; + if let Err(err) = run_decimals_backfill_pass(&provider, &db, &network, concurrency).await { + tracing::error!(?err, "balancer token decimals backfill pass failed"); + } + } +} + +async fn run_decimals_backfill_pass( + provider: &AlloyProvider, + db: &PgPool, + network: &NetworkName, + concurrency: usize, +) -> Result<()> { + let tokens = db::get_tokens_missing_decimals(db).await?; + let network = network.as_str(); + crate::metrics::Metrics::get() + .backfill_pending + .with_label_values(&[network, "decimals"]) + .set(i64::try_from(tokens.len()).unwrap_or(-1)); + if tokens.is_empty() { + return Ok(()); + } + let total = tokens.len(); + tracing::info!(total, "backfilling balancer token decimals"); + + let mut stream = futures::stream::iter(tokens) + .map(|token| async move { + // `None` → `-1` is the "tried, failed" sentinel; the next pass's + // `IS NULL` filter skips it. + let decimals = super::fetch_decimals(provider, token) + .await + .map(i16::from) + .unwrap_or(-1); + (token, decimals) + }) + .buffer_unordered(concurrency) + .ready_chunks(BACKFILL_BATCH_SIZE); + + let mut updated = 0usize; + while let Some(batch) = stream.next().await { + match write_decimals_batch(db, &batch).await { + Ok(()) => { + for (_, decimals) in &batch { + updated += 1; + let result = if *decimals < 0 { "empty" } else { "ok" }; + crate::metrics::Metrics::get() + .backfilled + .with_label_values(&[network, "decimals", result]) + .inc(); + } + } + Err(err) => tracing::warn!(?err, "failed to backfill balancer decimals batch"), + } + } + tracing::info!(updated, total, "balancer token decimals backfill complete"); + Ok(()) +} + +async fn write_decimals_batch(db: &PgPool, entries: &[(Address, i16)]) -> Result<()> { + let mut tx = db.begin().await.context("begin decimals batch tx")?; + db::batch_set_token_decimals(&mut tx, entries).await?; + tx.commit().await.context("commit decimals batch tx")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const POOL: Address = Address::repeat_byte(0x11); + const TOKEN0: Address = Address::repeat_byte(0x01); + const TOKEN1: Address = Address::repeat_byte(0x02); + // 0.5 in Balancer's 1e18 fixed point. + const HALF: u64 = 500_000_000_000_000_000; + + #[test] + fn assembles_weighted_pool_with_fractional_weights() { + let pool_id = B256::repeat_byte(0x22); + // TOKEN1 deliberately missing from the decimals cache. + let decimals = HashMap::from([(TOKEN0, 18u8)]); + let weights = Some(vec![U256::from(HALF), U256::from(HALF)]); + + let pool = assemble_pool( + POOL, + pool_id, + PoolType::Weighted, + 100, + vec![TOKEN0, TOKEN1], + &decimals, + weights, + ); + + assert_eq!(pool.pool_id, pool_id); + assert_eq!(pool.address, POOL); + assert_eq!(pool.created_block, 100); + assert_eq!(pool.tokens.len(), 2); + + assert_eq!(pool.tokens[0].position, 0); + assert_eq!(pool.tokens[0].address, TOKEN0); + assert_eq!(pool.tokens[0].decimals, Some(18)); + assert_eq!( + pool.tokens[0].weight.as_ref().unwrap().to_string(), + "0.500000000000000000" + ); + + assert_eq!(pool.tokens[1].position, 1); + assert_eq!(pool.tokens[1].decimals, None); + assert!(pool.tokens[1].weight.is_some()); + } + + #[test] + fn assembles_non_weighted_pool_without_weights() { + let decimals = HashMap::from([(TOKEN0, 6u8)]); + let pool = assemble_pool( + POOL, + B256::repeat_byte(0x22), + PoolType::Stable, + 100, + vec![TOKEN0], + &decimals, + None, + ); + assert_eq!(pool.tokens.len(), 1); + assert_eq!(pool.tokens[0].decimals, Some(6)); + assert!(pool.tokens[0].weight.is_none()); + } + + #[test] + fn configured_factories_maps_groups_to_pool_types() { + let f = |b| FactoryConfig { + address: Address::ZERO, + deploy_block: b, + }; + let config = BalancerV2Config { + vault: Address::ZERO, + chunk_size: 100_000, + weighted: vec![f(1)], + weighted_v3plus: vec![f(2)], + stable: vec![f(3)], + liquidity_bootstrapping: vec![f(4)], + composable_stable: vec![f(5)], + }; + let got: Vec<_> = configured_factories(&config) + .into_iter() + .map(|(t, factory)| (t, factory.deploy_block)) + .collect(); + assert_eq!( + got, + vec![ + (PoolType::Weighted, 1), + (PoolType::Weighted, 2), + (PoolType::Stable, 3), + (PoolType::LiquidityBootstrapping, 4), + (PoolType::ComposableStable, 5), + ] + ); + } +} diff --git a/crates/pool-indexer/src/indexer/mod.rs b/crates/pool-indexer/src/indexer/mod.rs index c24144d0e0..b6afd520ab 100644 --- a/crates/pool-indexer/src/indexer/mod.rs +++ b/crates/pool-indexer/src/indexer/mod.rs @@ -1 +1,159 @@ +pub mod balancer_v2; pub mod uniswap_v3; + +use { + alloy_primitives::{Address, B256}, + alloy_provider::Provider, + alloy_rpc_types_eth::{Filter, FilterSet, Log}, + alloy_transport::RpcError, + anyhow::Result, + contracts::ERC20, + ethrpc::{AlloyProvider, alloy::errors::ContractErrorExt}, + std::time::Duration, +}; + +/// Retries `f` while the error is a transient transport failure +/// (`is_node_error`). Contract reverts and decoding failures bail out +/// immediately. On giveup, hands `on_giveup` the accumulated errors and +/// returns `None`. +pub(crate) async fn retry_node_call( + f: impl Fn() -> Fut, + on_giveup: impl FnOnce(&[alloy_contract::Error]), +) -> Option +where + Fut: std::future::Future>, +{ + match shared::retry::retry_with_sleep_if(f, |err: &alloy_contract::Error| err.is_node_error()) + .await + { + Ok(v) => Some(v), + Err(errors) => { + on_giveup(&errors); + None + } + } +} + +/// Fetches a token's `decimals()`, retrying transient node errors. Returns +/// `None` if the call ultimately fails (revert / decode / giveup). +pub(crate) async fn fetch_decimals(provider: &AlloyProvider, token: Address) -> Option { + retry_node_call( + || async move { + ERC20::Instance::new(token, provider.clone()) + .decimals() + .call() + .await + }, + |errors| tracing::warn!(%token, ?errors, "fetch_decimals gave up"), + ) + .await +} + +/// True if the server-side JSON-RPC payload rejected `eth_getLogs` for +/// being too wide / returning too many logs / exceeding a response-size +/// cap / hitting the server's query timeout. Substrings cover the +/// rejections empirically seen on OVH and Alchemy mainnet. Transport-level +/// errors (HTTP timeouts, DNS, connection resets) live in other `RpcError` +/// variants and short-circuit to false, so client-side noise can't trigger +/// pointless bisection. +pub(crate) fn is_range_too_large(err: &alloy_transport::TransportError) -> bool { + let RpcError::ErrorResp(payload) = err else { + return false; + }; + let msg = payload.message.to_lowercase(); + msg.contains("max block range") + || msg.contains("max results") + || msg.contains("log response size exceeded") + || msg.contains("query timeout exceeded") + || msg.contains("response is too big") +} + +/// Bisecting bound — substring matching on RPC error messages is necessarily +/// approximate, and a misclassified error would otherwise burn `log2(range)` +/// RPC calls before the recursion bottoms out at `to == from`. 8 halvings = +/// 256× resolution; for the indexer's ~1k-block chunks that means giving up +/// around ~4-block ranges, well past where range-size could plausibly still +/// be the cause. +const MAX_BISECTION_DEPTH: u32 = 8; + +/// Retry transient `eth_getLogs` failures (timeout, reset, throttle) with +/// backoff, capped by a per-call timeout, so one blip can't abort a long +/// cold-seed scan. Range-size rejections are bisected, not retried. +const GETLOGS_TIMEOUT: Duration = Duration::from_secs(45); +const MAX_GETLOGS_RETRIES: u32 = 6; +const GETLOGS_RETRY_BACKOFF: Duration = Duration::from_secs(2); + +/// Fetches logs for `[from, to]` filtered by the given contract addresses +/// and `topic0` event signatures, sequentially bisecting the block range on +/// "too large" rejections until each sub-range is tractable. An empty +/// `addresses` list means "any contract". Bisection depth is capped by +/// [`MAX_BISECTION_DEPTH`]. +pub(crate) fn bisecting_get_logs( + provider: &AlloyProvider, + from: u64, + to: u64, + addresses: Vec
, + topics: Vec, +) -> futures::future::BoxFuture<'_, Result>> { + bisecting_get_logs_with_depth(provider, from, to, addresses, topics, 0) +} + +fn bisecting_get_logs_with_depth( + provider: &AlloyProvider, + from: u64, + to: u64, + addresses: Vec
, + topics: Vec, + depth: u32, +) -> futures::future::BoxFuture<'_, Result>> { + Box::pin(async move { + let filter = Filter::new() + .address(addresses.clone()) + .event_signature(FilterSet::from_iter(topics.clone())) + .from_block(from) + .to_block(to); + + let mut attempt = 0u32; + loop { + let err = match tokio::time::timeout(GETLOGS_TIMEOUT, provider.get_logs(&filter)).await + { + Ok(Ok(logs)) => return Ok(logs), + // Range-size rejection: bisect (below), not retried. + Ok(Err(err)) if is_range_too_large(&err) => { + if to <= from || depth >= MAX_BISECTION_DEPTH { + return Err( + anyhow::Error::new(err).context(format!("get_logs({from}..={to})")) + ); + } + break; + } + Ok(Err(err)) => anyhow::Error::new(err).context(format!("get_logs({from}..={to})")), + Err(_elapsed) => anyhow::anyhow!("get_logs({from}..={to}) timed out"), + }; + // Transient failure: retry with backoff, then give up. + if attempt >= MAX_GETLOGS_RETRIES { + return Err(err); + } + attempt += 1; + tracing::warn!(%err, attempt, from, to, "get_logs failed, retrying"); + tokio::time::sleep(GETLOGS_RETRY_BACKOFF * attempt).await; + } + + let mid = (from + to) / 2; + tracing::debug!(from, to, mid, depth, "range too large, bisecting"); + let mut left = bisecting_get_logs_with_depth( + provider, + from, + mid, + addresses.clone(), + topics.clone(), + depth + 1, + ) + .await?; + let right = + bisecting_get_logs_with_depth(provider, mid + 1, to, addresses, topics, depth + 1) + .await?; + left.extend(right); + Ok(left) + }) +} diff --git a/crates/pool-indexer/src/indexer/uniswap_v3.rs b/crates/pool-indexer/src/indexer/uniswap_v3.rs index 72bbbae44a..6a6c6f3745 100644 --- a/crates/pool-indexer/src/indexer/uniswap_v3.rs +++ b/crates/pool-indexer/src/indexer/uniswap_v3.rs @@ -1,20 +1,19 @@ use { crate::{ config::{IndexerConfig, NetworkName}, - db::uniswap_v3 as db, + db::{get_checkpoint, set_checkpoint, uniswap_v3 as db}, }, - alloy_primitives::{Address, B256, aliases::U160}, + alloy_primitives::{Address, aliases::U160}, alloy_provider::Provider, - alloy_rpc_types_eth::{BlockNumberOrTag, Filter, FilterSet, Log}, + alloy_rpc_types_eth::{BlockNumberOrTag, Log}, alloy_sol_types::SolEvent, - alloy_transport::RpcError, anyhow::{Context, Result}, contracts::{ ERC20, IUniswapV3Factory::IUniswapV3Factory::PoolCreated, UniswapV3Pool::UniswapV3Pool::{Burn, Initialize, Mint, Swap}, }, - ethrpc::{AlloyProvider, alloy::errors::ContractErrorExt}, + ethrpc::AlloyProvider, futures::{StreamExt, TryStreamExt}, itertools::Itertools, sqlx::PgPool, @@ -153,7 +152,7 @@ impl UniswapV3Indexer { /// Bails if a checkpoint already exists; overwriting would silently /// regress and re-index history. pub async fn catch_up(&self, from_block: u64) -> Result<()> { - if db::get_checkpoint(&self.db, &self.factory).await?.is_some() { + if get_checkpoint(&self.db, &self.factory).await?.is_some() { anyhow::bail!( "catch_up called but checkpoint already exists for chain {} factory {}", self.chain_id, @@ -161,7 +160,7 @@ impl UniswapV3Indexer { ); } let mut tx = self.db.begin().await.context("begin checkpoint tx")?; - db::set_checkpoint(&mut tx, &self.factory, from_block).await?; + set_checkpoint(&mut tx, &self.factory, from_block).await?; tx.commit().await.context("commit checkpoint tx")?; self.catch_up_to_finalized().await @@ -228,9 +227,7 @@ impl UniswapV3Indexer { } async fn last_indexed_block(&self) -> Result { - Ok(db::get_checkpoint(&self.db, &self.factory) - .await? - .unwrap_or(0)) + Ok(get_checkpoint(&self.db, &self.factory).await?.unwrap_or(0)) } fn pending_chunks(&self, last_indexed_block: u64, finalized_block: u64) -> Vec { @@ -263,7 +260,7 @@ impl UniswapV3Indexer { // // The `WHERE EXISTS (… uniswap_v3_pools …)` clauses in the batch // writers stay as defense-in-depth. - bisecting_get_logs( + super::bisecting_get_logs( &self.provider, from, to, @@ -360,7 +357,7 @@ impl UniswapV3Indexer { db::upsert_pool_states(&mut tx, &self.factory, &changes.pool_states).await?; db::batch_update_pool_liquidity(&mut tx, &self.factory, &changes.liquidity_updates).await?; db::batch_update_ticks(&mut tx, &self.factory, &changes.tick_deltas).await?; - db::set_checkpoint(&mut tx, &self.factory, chunk.end).await?; + set_checkpoint(&mut tx, &self.factory, chunk.end).await?; tx.commit().await.context("commit transaction")?; Ok(()) @@ -371,7 +368,7 @@ impl UniswapV3Indexer { async fn prefetch_decimals(&self, logs: &[Log]) -> DecimalsCache { futures::stream::iter(pool_created_token_addresses(self.factory, logs)) .map(|token| async move { - let dec = fetch_decimals(&self.provider, token).await; + let dec = super::fetch_decimals(&self.provider, token).await; (token, dec) }) .buffer_unordered(self.prefetch_concurrency) @@ -381,28 +378,6 @@ impl UniswapV3Indexer { } } -/// Retries `f` while the error is a transient transport failure -/// (`is_node_error`). Contract reverts and decoding failures bail out -/// immediately. On giveup, hands `on_giveup` the accumulated errors and -/// returns `None`. -async fn retry_node_call( - f: impl Fn() -> Fut, - on_giveup: impl FnOnce(&[alloy_contract::Error]), -) -> Option -where - Fut: std::future::Future>, -{ - match shared::retry::retry_with_sleep_if(f, |err: &alloy_contract::Error| err.is_node_error()) - .await - { - Ok(v) => Some(v), - Err(errors) => { - on_giveup(&errors); - None - } - } -} - /// Unique addresses with a `Mint` or `Burn` in the chunk. We pre-load /// their `(tick, liquidity)` so the post-event liquidity can be derived /// from the in-event delta without an extra RPC. @@ -433,19 +408,6 @@ fn pool_event_emitters(logs: &[Log]) -> Vec
{ .collect() } -async fn fetch_decimals(provider: &AlloyProvider, token: Address) -> Option { - retry_node_call( - || async move { - ERC20::Instance::new(token, provider.clone()) - .decimals() - .call() - .await - }, - |errors| tracing::warn!(%token, ?errors, "fetch_decimals gave up"), - ) - .await -} - /// Periodically fills `token{0,1}_symbol` on newly-indexed pools. /// /// Tokens whose `symbol()` call fails (revert / decode / empty) are stored @@ -586,7 +548,7 @@ async fn run_decimals_backfill_pass( .map(|token| async move { // `None` → `-1` is the "tried and failed" sentinel; the // `IS NULL` filter on the next pass skips it. - let dec = fetch_decimals(provider, token) + let dec = super::fetch_decimals(provider, token) .await .map(i16::from) .unwrap_or(-1); @@ -655,115 +617,6 @@ async fn fetch_symbol(provider: &AlloyProvider, token: Address) -> Option bool { - let RpcError::ErrorResp(payload) = err else { - return false; - }; - let msg = payload.message.to_lowercase(); - msg.contains("max block range") - || msg.contains("max results") - || msg.contains("log response size exceeded") - || msg.contains("query timeout exceeded") - || msg.contains("response is too big") -} - -/// Bisecting bound — substring matching on RPC error messages is necessarily -/// approximate, and a misclassified error would otherwise burn `log2(range)` -/// RPC calls before the recursion bottoms out at `to == from`. 8 halvings = -/// 256× resolution; for the indexer's ~1k-block chunks that means giving up -/// around ~4-block ranges, well past where range-size could plausibly still -/// be the cause. -const MAX_BISECTION_DEPTH: u32 = 8; - -/// Retry transient `eth_getLogs` failures (timeout, reset, throttle) with -/// backoff, capped by a per-call timeout, so one blip can't abort a long -/// cold-seed scan. Range-size rejections are bisected, not retried. -const GETLOGS_TIMEOUT: Duration = Duration::from_secs(45); -const MAX_GETLOGS_RETRIES: u32 = 6; -const GETLOGS_RETRY_BACKOFF: Duration = Duration::from_secs(2); - -/// Fetches logs for `[from, to]` filtered by the given contract addresses -/// and `topic0` event signatures, sequentially bisecting the block range on -/// "too large" rejections until each sub-range is tractable. An empty -/// `addresses` list means "any contract". Bisection depth is capped by -/// [`MAX_BISECTION_DEPTH`]. -pub(crate) fn bisecting_get_logs( - provider: &AlloyProvider, - from: u64, - to: u64, - addresses: Vec
, - topics: Vec, -) -> futures::future::BoxFuture<'_, Result>> { - bisecting_get_logs_with_depth(provider, from, to, addresses, topics, 0) -} - -fn bisecting_get_logs_with_depth( - provider: &AlloyProvider, - from: u64, - to: u64, - addresses: Vec
, - topics: Vec, - depth: u32, -) -> futures::future::BoxFuture<'_, Result>> { - Box::pin(async move { - let filter = Filter::new() - .address(addresses.clone()) - .event_signature(FilterSet::from_iter(topics.clone())) - .from_block(from) - .to_block(to); - - let mut attempt = 0u32; - loop { - let err = match tokio::time::timeout(GETLOGS_TIMEOUT, provider.get_logs(&filter)).await - { - Ok(Ok(logs)) => return Ok(logs), - // Range-size rejection: bisect (below), not retried. - Ok(Err(err)) if is_range_too_large(&err) => { - if to <= from || depth >= MAX_BISECTION_DEPTH { - return Err( - anyhow::Error::new(err).context(format!("get_logs({from}..={to})")) - ); - } - break; - } - Ok(Err(err)) => anyhow::Error::new(err).context(format!("get_logs({from}..={to})")), - Err(_elapsed) => anyhow::anyhow!("get_logs({from}..={to}) timed out"), - }; - // Transient failure: retry with backoff, then give up. - if attempt >= MAX_GETLOGS_RETRIES { - return Err(err); - } - attempt += 1; - tracing::warn!(%err, attempt, from, to, "get_logs failed, retrying"); - tokio::time::sleep(GETLOGS_RETRY_BACKOFF * attempt).await; - } - - let mid = (from + to) / 2; - tracing::debug!(from, to, mid, depth, "range too large, bisecting"); - let mut left = bisecting_get_logs_with_depth( - provider, - from, - mid, - addresses.clone(), - topics.clone(), - depth + 1, - ) - .await?; - let right = - bisecting_get_logs_with_depth(provider, mid + 1, to, addresses, topics, depth + 1) - .await?; - left.extend(right); - Ok(left) - }) -} - /// Collects the unique set of token addresses from all `PoolCreated` events /// emitted by `factory` in `logs`. fn pool_created_token_addresses( diff --git a/crates/pool-indexer/src/run.rs b/crates/pool-indexer/src/run.rs index 44d85d59f8..0810445a91 100644 --- a/crates/pool-indexer/src/run.rs +++ b/crates/pool-indexer/src/run.rs @@ -2,8 +2,11 @@ use { crate::{ api::AppState, arguments::Arguments, - config::{Configuration, NetworkConfig}, - indexer::uniswap_v3::UniswapV3Indexer, + config::{BalancerV2Config, Configuration, FactoryConfig, NetworkConfig}, + indexer::{ + balancer_v2::{self, BalancerV2Indexer, PoolType}, + uniswap_v3::UniswapV3Indexer, + }, }, alloy_provider::Provider, clap::Parser, @@ -61,6 +64,21 @@ pub async fn bootstrap(config: Configuration) { }); } } + if let Some(balancer) = &network.balancer_v2 { + for (pool_type, factory) in balancer_v2::configured_factories(balancer) { + let indexer = BalancerV2Indexer::new( + provider.clone(), + db.clone(), + balancer_indexer_config(&network, balancer, pool_type, factory), + ); + factory_set.spawn(async move { + indexer + .bootstrap() + .await + .expect("balancer bootstrap failed"); + }); + } + } while let Some(result) = factory_set.join_next().await { result.expect("bootstrap task panicked"); } @@ -78,7 +96,12 @@ pub async fn run(config: Configuration) { .network .uniswap_v3 .as_ref() - .map_or(0, |u| u.factories.len()), + .map_or(0, |u| u.factories.len()) + + config + .network + .balancer_v2 + .as_ref() + .map_or(0, |b| b.factory_count()), )); // Abort the metrics task when `run` exits, so tests can rebind the port. @@ -178,10 +201,11 @@ async fn run_network_indexer(db: PgPool, network: NetworkConfig, barrier: Arc balancer_v2::IndexerConfig { + balancer_v2::IndexerConfig { + network: network.name.clone(), + vault: balancer.vault, + factory: factory.address, + pool_type, + deploy_block: factory.deploy_block, + chunk_size: balancer.chunk_size, + use_latest: network.use_latest, + fetch_concurrency: network.fetch_concurrency, + enrich_concurrency: network.prefetch_concurrency, + } +} + async fn run_factory_indexer( db: PgPool, indexer: UniswapV3Indexer, @@ -258,7 +325,7 @@ async fn bootstrap_factory( network: &NetworkConfig, factory: &crate::config::FactoryConfig, ) { - let checkpoint = crate::db::uniswap_v3::get_checkpoint(db, &factory.address) + let checkpoint = crate::db::get_checkpoint(db, &factory.address) .await .expect("failed to read checkpoint"); match checkpoint { From 7fbbe456ff9f183528151280bf1c569e529b7076 Mon Sep 17 00:00:00 2001 From: Aryan Godara Date: Thu, 27 Aug 2026 16:28:48 +0530 Subject: [PATCH 5/7] add bal-v2 pools API to pool-indexer --- .../pool-indexer/src/api/balancer_v2/mod.rs | 118 ++++++++++++++++ .../src/api/balancer_v2/pools_by_ids.rs | 36 +++++ .../src/api/balancer_v2/pools_list.rs | 58 ++++++++ crates/pool-indexer/src/api/mod.rs | 16 ++- crates/pool-indexer/src/api/routes.rs | 8 +- .../src/api/uniswap_v3/bulk_ticks.rs | 2 +- .../src/api/uniswap_v3/pool_ticks.rs | 2 +- .../src/api/uniswap_v3/pools_by_ids.rs | 2 +- .../src/api/uniswap_v3/pools_list.rs | 2 +- crates/pool-indexer/src/db/balancer_v2.rs | 129 +++++++++++++++++- crates/pool-indexer/src/db/mod.rs | 30 +++- crates/pool-indexer/src/db/uniswap_v3.rs | 23 ---- crates/pool-indexer/src/run.rs | 8 +- 13 files changed, 396 insertions(+), 38 deletions(-) create mode 100644 crates/pool-indexer/src/api/balancer_v2/mod.rs create mode 100644 crates/pool-indexer/src/api/balancer_v2/pools_by_ids.rs create mode 100644 crates/pool-indexer/src/api/balancer_v2/pools_list.rs diff --git a/crates/pool-indexer/src/api/balancer_v2/mod.rs b/crates/pool-indexer/src/api/balancer_v2/mod.rs new file mode 100644 index 0000000000..52e03cbf1f --- /dev/null +++ b/crates/pool-indexer/src/api/balancer_v2/mod.rs @@ -0,0 +1,118 @@ +pub mod pools_by_ids; +pub mod pools_list; + +use { + crate::db::balancer_v2 as db, + alloy_primitives::{Address, B256}, + axum::{ + Json, + response::{IntoResponse, Response}, + }, + serde::{Deserialize, Deserializer, Serialize}, +}; +pub use {pools_by_ids::get_pools_by_ids, pools_list::get_pools}; + +/// Max pool ids per bulk lookup. Keeps URLs under proxy limits and caps the DB +/// query size. +pub(super) const MAX_POOL_IDS_PER_REQUEST: usize = 500; + +/// Deserializes `?pool_ids=0x…,0x…` into 32-byte pool ids. Parsing and the cap +/// happen in the extractor so handlers see a `Vec`. +pub(crate) struct PoolIds(pub Vec); + +impl<'de> Deserialize<'de> for PoolIds { + fn deserialize>(de: D) -> Result { + let raw = String::deserialize(de)?; + let out: Vec = raw + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|entry| { + entry + .parse::() + .map_err(|_| serde::de::Error::custom("invalid pool id")) + }) + .collect::>()?; + if out.len() > MAX_POOL_IDS_PER_REQUEST { + return Err(serde::de::Error::custom(format!( + "too many pool ids; max {MAX_POOL_IDS_PER_REQUEST}" + ))); + } + Ok(PoolIds(out)) + } +} + +/// One token of a pool, in `getPoolTokens` order. +#[derive(Serialize)] +pub struct TokenInfo { + pub address: Address, + pub decimals: u8, + /// Normalized weight as a decimal fraction (e.g. "0.5"); present only for + /// weighted pools. + #[serde(skip_serializing_if = "Option::is_none")] + pub weight: Option, +} + +/// A single Balancer V2 pool. Field names mirror the driver's discovery type. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PoolResponse { + pub pool_type: String, + pub id: B256, + pub address: Address, + pub factory: Address, + /// Discovery-time flag; the driver re-checks it on-chain for LBP pools. + /// Always `true` here, since the indexer stores static metadata only. + pub swap_enabled: bool, + pub tokens: Vec, +} + +#[derive(Serialize)] +pub struct PoolsResponse { + /// Latest block every configured balancer factory is indexed through. + pub block_number: u64, + pub pools: Vec, + /// Pass as `after=` to fetch the next page; `null` on the last page. + pub next_cursor: Option, +} + +impl From for TokenInfo { + fn from(t: db::BalancerTokenRow) -> Self { + Self { + address: t.address, + decimals: t.decimals, + // `normalized()` strips the trailing zeros Postgres NUMERIC pads on + // when decoded through its base-10000 digit groups. Without it a + // weight can render with >18 fractional digits, which the driver's + // fixed-point parser rejects. + weight: t.weight.map(|w| w.normalized().to_string()), + } + } +} + +impl From for PoolResponse { + fn from(row: db::BalancerPoolRow) -> Self { + Self { + pool_type: row.pool_type, + id: row.pool_id, + address: row.address, + factory: row.factory, + swap_enabled: true, + tokens: row.tokens.into_iter().map(TokenInfo::from).collect(), + } + } +} + +/// Shared `PoolsResponse` builder for the balancer listing endpoints. +pub(super) fn pools_response( + block_number: u64, + pools: Vec, + next_cursor: Option, +) -> Response { + Json(PoolsResponse { + block_number, + pools: pools.into_iter().map(PoolResponse::from).collect(), + next_cursor, + }) + .into_response() +} diff --git a/crates/pool-indexer/src/api/balancer_v2/pools_by_ids.rs b/crates/pool-indexer/src/api/balancer_v2/pools_by_ids.rs new file mode 100644 index 0000000000..1a55346798 --- /dev/null +++ b/crates/pool-indexer/src/api/balancer_v2/pools_by_ids.rs @@ -0,0 +1,36 @@ +//! `GET /api/v1/{network}/balancer/v2/pools/by-ids?pool_ids=…` + +use { + super::{PoolIds, pools_response}, + crate::{ + api::{ApiError, AppState, latest_indexed_block}, + db::balancer_v2 as db, + }, + axum::{ + extract::{Query, State}, + response::Response, + }, + serde::Deserialize, + std::sync::Arc, +}; + +#[derive(Deserialize)] +pub struct BulkLookupQuery { + /// Comma-separated 32-byte pool ids. Capped at + /// [`super::MAX_POOL_IDS_PER_REQUEST`]; clients with more should chunk. + pub pool_ids: PoolIds, +} + +/// Pools matching `pool_ids`, sorted by `pool_id`. Unknown ids are skipped +/// silently; treat a partial response as "these are the ones I have". +/// +/// `block_number` is read first so the envelope is never *newer* than the row +/// data (the indexer can advance between the two reads, never regress). +pub async fn get_pools_by_ids( + State(state): State>, + Query(BulkLookupQuery { pool_ids }): Query, +) -> Result { + let block = latest_indexed_block(&state.db, &state.balancer_v2_factories).await?; + let pools = db::get_pools_by_ids(&state.db, &pool_ids.0).await?; + Ok(pools_response(block, pools, None)) +} diff --git a/crates/pool-indexer/src/api/balancer_v2/pools_list.rs b/crates/pool-indexer/src/api/balancer_v2/pools_list.rs new file mode 100644 index 0000000000..75784ee21d --- /dev/null +++ b/crates/pool-indexer/src/api/balancer_v2/pools_list.rs @@ -0,0 +1,58 @@ +//! `GET /api/v1/{network}/balancer/v2/pools` — cursor-paginated pool list. + +use { + super::pools_response, + crate::{ + api::{ApiError, AppState, latest_indexed_block}, + db::balancer_v2 as db, + }, + alloy_primitives::B256, + axum::{ + extract::{Query, State}, + response::Response, + }, + serde::Deserialize, + std::sync::Arc, +}; + +const DEFAULT_PAGE_LIMIT: u64 = 1_000; + +/// Hard server-side cap on `limit`. Applied even if the client asks for more. +const MAX_PAGE_LIMIT: u64 = 5_000; + +#[derive(Deserialize)] +pub struct ListPoolsQuery { + /// Cursor from the previous page (the last-seen `pool_id`); omit to start + /// from the beginning. + pub after: Option, + /// Clamped to `[1, MAX_PAGE_LIMIT]`; defaults to `DEFAULT_PAGE_LIMIT`. + pub limit: Option, +} + +impl ListPoolsQuery { + fn page_limit(&self) -> u64 { + self.limit + .unwrap_or(DEFAULT_PAGE_LIMIT) + .clamp(1, MAX_PAGE_LIMIT) + } +} + +/// All indexed pools, sorted by `pool_id`. +pub async fn get_pools( + State(state): State>, + Query(query): Query, +) -> Result { + let block_number = latest_indexed_block(&state.db, &state.balancer_v2_factories).await?; + let limit = query.page_limit(); + let cursor = query.after.map(|id| id.as_slice().to_vec()); + + let mut rows = db::get_pools(&state.db, cursor, limit + 1).await?; + + let has_next = rows.len() > limit as usize; + rows.truncate(limit as usize); + let next_cursor = has_next + .then(|| rows.last().map(|row| format!("{:#x}", row.pool_id))) + .flatten(); + + Ok(pools_response(block_number, rows, next_cursor)) +} diff --git a/crates/pool-indexer/src/api/mod.rs b/crates/pool-indexer/src/api/mod.rs index 2dc462acc2..d59165ee0b 100644 --- a/crates/pool-indexer/src/api/mod.rs +++ b/crates/pool-indexer/src/api/mod.rs @@ -1,3 +1,4 @@ +pub mod balancer_v2; pub mod routes; pub mod uniswap_v3; @@ -19,9 +20,11 @@ pub struct AppState { /// The network this process indexes. Requests whose `{network}` path /// segment doesn't match get a 404. pub network: NetworkName, - /// Configured factory addresses; the served envelope block is scoped to - /// these so a removed factory's stale checkpoint can't pin it. - pub factories: BTreeSet
, + /// Uniswap V3 factories. The served envelope block is scoped to these so a + /// removed factory's stale checkpoint can't pin it. + pub uniswap_v3_factories: BTreeSet
, + /// Balancer V2 factories, scoping the balancer envelope block likewise. + pub balancer_v2_factories: BTreeSet
, } impl AppState { @@ -62,8 +65,11 @@ impl From for ApiError { } } -pub(super) async fn latest_indexed_block(state: &AppState) -> Result { - crate::db::uniswap_v3::get_latest_indexed_block(&state.db, &state.factories) +pub(super) async fn latest_indexed_block( + db: &PgPool, + factories: &BTreeSet
, +) -> Result { + crate::db::get_latest_indexed_block(db, factories) .await? .ok_or(ApiError::NotReady) } diff --git a/crates/pool-indexer/src/api/routes.rs b/crates/pool-indexer/src/api/routes.rs index 8e95733e20..69fa4e20e4 100644 --- a/crates/pool-indexer/src/api/routes.rs +++ b/crates/pool-indexer/src/api/routes.rs @@ -1,7 +1,7 @@ //! HTTP routing for the pool-indexer API. use { - super::{ApiError, AppState, uniswap_v3}, + super::{ApiError, AppState, balancer_v2, uniswap_v3}, axum::{ Router, extract::{MatchedPath, Path, Request, State}, @@ -26,9 +26,15 @@ pub fn router(state: Arc) -> Router { .route("/pools/{pool_address}/ticks", get(uniswap_v3::get_ticks)) .route_layer(middleware::from_fn_with_state(state.clone(), network_guard)); + let balancer_v2_routes = Router::new() + .route("/pools", get(balancer_v2::get_pools)) + .route("/pools/by-ids", get(balancer_v2::get_pools_by_ids)) + .route_layer(middleware::from_fn_with_state(state.clone(), network_guard)); + Router::new() .route("/health", get(health)) .nest("/api/v1/{network}/uniswap/v3", v3_routes) + .nest("/api/v1/{network}/balancer/v2", balancer_v2_routes) .with_state(state) .layer(middleware::from_fn(record_request_metrics)) .layer( diff --git a/crates/pool-indexer/src/api/uniswap_v3/bulk_ticks.rs b/crates/pool-indexer/src/api/uniswap_v3/bulk_ticks.rs index aaeb8539e3..84a7f867c1 100644 --- a/crates/pool-indexer/src/api/uniswap_v3/bulk_ticks.rs +++ b/crates/pool-indexer/src/api/uniswap_v3/bulk_ticks.rs @@ -44,7 +44,7 @@ pub async fn get_ticks_bulk( Query(BulkTicksQuery { pool_ids }): Query, ) -> Result { let (block, ticks) = tokio::join!( - latest_indexed_block(&state), + latest_indexed_block(&state.db, &state.uniswap_v3_factories), db::get_ticks_for_pools(&state.db, &pool_ids.0), ); diff --git a/crates/pool-indexer/src/api/uniswap_v3/pool_ticks.rs b/crates/pool-indexer/src/api/uniswap_v3/pool_ticks.rs index bc2ebe97a8..9c40756a42 100644 --- a/crates/pool-indexer/src/api/uniswap_v3/pool_ticks.rs +++ b/crates/pool-indexer/src/api/uniswap_v3/pool_ticks.rs @@ -28,7 +28,7 @@ pub async fn get_ticks( Path((_network, pool)): Path<(String, Address)>, ) -> Result { let (block, ticks) = tokio::join!( - latest_indexed_block(&state), + latest_indexed_block(&state.db, &state.uniswap_v3_factories), db::get_ticks(&state.db, &pool), ); diff --git a/crates/pool-indexer/src/api/uniswap_v3/pools_by_ids.rs b/crates/pool-indexer/src/api/uniswap_v3/pools_by_ids.rs index 079418d31e..7a7c268d89 100644 --- a/crates/pool-indexer/src/api/uniswap_v3/pools_by_ids.rs +++ b/crates/pool-indexer/src/api/uniswap_v3/pools_by_ids.rs @@ -31,7 +31,7 @@ pub async fn get_pools_by_ids( State(state): State>, Query(BulkLookupQuery { pool_ids }): Query, ) -> Result { - let block = latest_indexed_block(&state).await?; + let block = latest_indexed_block(&state.db, &state.uniswap_v3_factories).await?; let pools = db::get_pools_by_ids(&state.db, &pool_ids.0).await?; Ok(pools_response(block, &pools, None)) } diff --git a/crates/pool-indexer/src/api/uniswap_v3/pools_list.rs b/crates/pool-indexer/src/api/uniswap_v3/pools_list.rs index 4b4aa3f5ec..55f6561151 100644 --- a/crates/pool-indexer/src/api/uniswap_v3/pools_list.rs +++ b/crates/pool-indexer/src/api/uniswap_v3/pools_list.rs @@ -42,7 +42,7 @@ pub async fn get_pools( State(state): State>, Query(query): Query, ) -> Result { - let block_number = latest_indexed_block(&state).await?; + let block_number = latest_indexed_block(&state.db, &state.uniswap_v3_factories).await?; let limit = query.page_limit(); let cursor = query.after.map(|addr| addr.as_slice().to_vec()); diff --git a/crates/pool-indexer/src/db/balancer_v2.rs b/crates/pool-indexer/src/db/balancer_v2.rs index c30674771d..0c2fa669a4 100644 --- a/crates/pool-indexer/src/db/balancer_v2.rs +++ b/crates/pool-indexer/src/db/balancer_v2.rs @@ -1,9 +1,13 @@ use { - crate::{db::bytes_to_addr, indexer::balancer_v2::NewBalancerPool}, - alloy_primitives::Address, + crate::{ + db::{bytes_to_addr, bytes_to_b256}, + indexer::balancer_v2::NewBalancerPool, + }, + alloy_primitives::{Address, B256}, anyhow::{Context, Result}, bigdecimal::BigDecimal, - sqlx::{PgPool, Postgres, Row, Transaction}, + sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow}, + std::collections::HashMap, }; /// Inserts discovered pools and their tokens. Pools are written before tokens @@ -114,3 +118,122 @@ pub async fn batch_set_token_decimals( .context("batch_set_token_decimals")?; Ok(()) } + +/// A discovered pool plus its `getPoolTokens`-ordered tokens, for the read API. +pub struct BalancerPoolRow { + pub pool_id: B256, + pub address: Address, + pub factory: Address, + pub pool_type: String, + pub tokens: Vec, +} + +/// One token of a pool, in registration order. `decimals` is always present: +/// pools with an unresolved-decimals token are excluded from the read path. +pub struct BalancerTokenRow { + pub address: Address, + pub decimals: u8, + pub weight: Option, +} + +/// Pools sorted by `pool_id`, paginated via `cursor` (the last-seen `pool_id`). +/// Only pools whose every token has resolved decimals are returned; a pool with +/// an unresolved token isn't servable (the driver requires `decimals`). +pub async fn get_pools( + pool: &PgPool, + cursor: Option>, + limit: u64, +) -> Result> { + let rows = sqlx::query( + "SELECT pool_id, address, factory, pool_type + FROM balancer_v2_pools p + WHERE ($1::BYTEA IS NULL OR pool_id > $1) + AND NOT EXISTS ( + SELECT 1 FROM balancer_v2_pool_tokens t + WHERE t.pool_id = p.pool_id AND (t.decimals IS NULL OR t.decimals < 0) + ) + ORDER BY pool_id + LIMIT $2", + ) + .bind(cursor) + .bind(limit.cast_signed()) + .fetch_all(pool) + .await + .context("balancer get_pools")?; + + assemble_pools(pool, rows).await +} + +/// Pools matching `pool_ids`, sorted by `pool_id`. Unknown ids and pools with +/// an unresolved-decimals token are skipped. +pub async fn get_pools_by_ids(pool: &PgPool, pool_ids: &[B256]) -> Result> { + if pool_ids.is_empty() { + return Ok(Vec::new()); + } + let ids: Vec<&[u8]> = pool_ids.iter().map(|id| id.as_slice()).collect(); + let rows = sqlx::query( + "SELECT pool_id, address, factory, pool_type + FROM balancer_v2_pools p + WHERE pool_id = ANY($1) + AND NOT EXISTS ( + SELECT 1 FROM balancer_v2_pool_tokens t + WHERE t.pool_id = p.pool_id AND (t.decimals IS NULL OR t.decimals < 0) + ) + ORDER BY pool_id", + ) + .bind(ids) + .fetch_all(pool) + .await + .context("balancer get_pools_by_ids")?; + + assemble_pools(pool, rows).await +} + +/// Loads the tokens for `pool_rows` in one query and attaches them in +/// `position` order. Callers restrict to pools with complete decimals, so each +/// `decimals` decodes as a plain `u8`. +async fn assemble_pools(pool: &PgPool, pool_rows: Vec) -> Result> { + if pool_rows.is_empty() { + return Ok(Vec::new()); + } + let pool_ids: Vec> = pool_rows.iter().map(|r| r.get("pool_id")).collect(); + let ids: Vec<&[u8]> = pool_ids.iter().map(|v| v.as_slice()).collect(); + + let token_rows = sqlx::query( + "SELECT pool_id, token, decimals, weight + FROM balancer_v2_pool_tokens + WHERE pool_id = ANY($1) + ORDER BY pool_id, position", + ) + .bind(ids) + .fetch_all(pool) + .await + .context("balancer pool tokens")?; + + let mut tokens: HashMap, Vec> = HashMap::new(); + for row in token_rows { + let decimals: i16 = row.get("decimals"); + tokens + .entry(row.get("pool_id")) + .or_default() + .push(BalancerTokenRow { + address: bytes_to_addr(row.get("token"))?, + decimals: u8::try_from(decimals).context("token decimals out of range")?, + weight: row.get("weight"), + }); + } + + pool_rows + .into_iter() + .map(|r| { + let pool_id: Vec = r.get("pool_id"); + Ok(BalancerPoolRow { + tokens: tokens.remove(&pool_id).unwrap_or_default(), + pool_id: bytes_to_b256(&pool_id)?, + address: bytes_to_addr(r.get("address"))?, + factory: bytes_to_addr(r.get("factory"))?, + pool_type: r.get("pool_type"), + }) + }) + .collect() +} diff --git a/crates/pool-indexer/src/db/mod.rs b/crates/pool-indexer/src/db/mod.rs index 433ed0f816..d8d5165957 100644 --- a/crates/pool-indexer/src/db/mod.rs +++ b/crates/pool-indexer/src/db/mod.rs @@ -2,9 +2,10 @@ pub mod balancer_v2; pub mod uniswap_v3; use { - alloy_primitives::Address, + alloy_primitives::{Address, B256}, anyhow::{Context, Result}, sqlx::{PgPool, Postgres, Row, Transaction}, + std::collections::BTreeSet, }; /// Decodes a Postgres `BYTEA` column into an [`Address`]. @@ -12,6 +13,11 @@ pub(crate) fn bytes_to_addr(b: Vec) -> Result
{ Address::try_from(b.as_slice()).context("invalid address bytes") } +/// Decodes a Postgres `BYTEA` column into a 32-byte [`B256`] Balancer pool id. +pub(crate) fn bytes_to_b256(b: &[u8]) -> Result { + B256::try_from(b).context("invalid pool_id bytes") +} + /// Highest block scanned for a factory's `PoolCreated` events. Shared by both /// indexers: `pool_indexer_checkpoints` is keyed by factory address, which is /// unique across protocols, so their rows never collide. @@ -44,3 +50,25 @@ pub async fn set_checkpoint( .context("set_checkpoint")?; Ok(()) } + +/// The block every configured factory is indexed through, so every served pool +/// is current at least to here. Scoped to `factories` so a decommissioned +/// factory's leftover checkpoint row can't pin the value. +pub async fn get_latest_indexed_block( + pool: &PgPool, + factories: &BTreeSet
, +) -> Result> { + let factories: Vec<&[u8]> = factories.iter().map(|f| f.as_slice()).collect(); + let row = sqlx::query( + "SELECT MIN(block_number) AS block FROM pool_indexer_checkpoints + WHERE contract_address = ANY($1)", + ) + .bind(factories) + .fetch_one(pool) + .await + .context("get_latest_indexed_block")?; + + Ok(row + .get::, _>("block") + .map(|b| b.cast_unsigned())) +} diff --git a/crates/pool-indexer/src/db/uniswap_v3.rs b/crates/pool-indexer/src/db/uniswap_v3.rs index cb195428e0..25efd5dcfa 100644 --- a/crates/pool-indexer/src/db/uniswap_v3.rs +++ b/crates/pool-indexer/src/db/uniswap_v3.rs @@ -9,7 +9,6 @@ use { num::ToPrimitive, number::conversions::u160_to_big_decimal, sqlx::{PgPool, Postgres, Row, Transaction, postgres::PgRow}, - std::collections::BTreeSet, }; fn address_bytes_list(addresses: &[Address]) -> Vec<&[u8]> { @@ -580,25 +579,3 @@ pub async fn batch_set_token_symbols( Ok(()) } - -/// The block every configured factory is indexed through, so every served pool -/// is current at least to here. Scoped to `factories` so a decommissioned -/// factory's leftover checkpoint row can't pin the value. -pub async fn get_latest_indexed_block( - pool: &PgPool, - factories: &BTreeSet
, -) -> Result> { - let factories: Vec<&[u8]> = factories.iter().map(|f| f.as_slice()).collect(); - let row = sqlx::query( - "SELECT MIN(block_number) AS block FROM pool_indexer_checkpoints - WHERE contract_address = ANY($1)", - ) - .bind(factories) - .fetch_one(pool) - .await - .context("get_latest_indexed_block")?; - - Ok(row - .get::, _>("block") - .map(|b| b.cast_unsigned())) -} diff --git a/crates/pool-indexer/src/run.rs b/crates/pool-indexer/src/run.rs index 0810445a91..5bf28119b7 100644 --- a/crates/pool-indexer/src/run.rs +++ b/crates/pool-indexer/src/run.rs @@ -182,12 +182,18 @@ fn build_api_state(db: &PgPool, network: &NetworkConfig) -> Arc { Arc::new(AppState { db: db.clone(), network: network.name.clone(), - factories: network + uniswap_v3_factories: network .uniswap_v3 .iter() .flat_map(|u| &u.factories) .map(|f| f.address) .collect(), + balancer_v2_factories: network + .balancer_v2 + .iter() + .flat_map(balancer_v2::configured_factories) + .map(|(_, factory)| factory.address) + .collect(), }) } From fed2f488f870df02451cb9c4a1ccc85091339754 Mon Sep 17 00:00:00 2001 From: Aryan Godara Date: Thu, 27 Aug 2026 17:57:19 +0530 Subject: [PATCH 6/7] replace balancer-v2 subgraph with pool-indexer in driver --- crates/driver/example.toml | 3 +- .../src/boundary/liquidity/balancer/v2/mod.rs | 8 +- crates/driver/src/infra/config/file/load.rs | 8 +- crates/driver/src/infra/config/file/mod.rs | 8 +- crates/driver/src/infra/liquidity/config.rs | 8 +- .../src/balancer_v2/graph_api.rs | 481 ------------------ .../liquidity-sources/src/balancer_v2/mod.rs | 19 +- .../src/balancer_v2/models.rs | 283 +++++++++++ .../src/balancer_v2/pool_fetching/mod.rs | 11 +- .../src/balancer_v2/pool_indexer.rs | 147 ++++++ .../src/balancer_v2/pool_init.rs | 27 - .../src/balancer_v2/pools/common.rs | 4 +- .../balancer_v2/pools/composable_stable.rs | 4 +- .../pools/liquidity_bootstrapping.rs | 4 +- .../src/balancer_v2/pools/mod.rs | 2 +- .../src/balancer_v2/pools/stable.rs | 4 +- .../src/balancer_v2/pools/weighted.rs | 4 +- crates/liquidity-sources/src/lib.rs | 1 - crates/liquidity-sources/src/macros.rs | 12 - crates/liquidity-sources/src/subgraph.rs | 303 ----------- 20 files changed, 479 insertions(+), 862 deletions(-) delete mode 100644 crates/liquidity-sources/src/balancer_v2/graph_api.rs create mode 100644 crates/liquidity-sources/src/balancer_v2/models.rs create mode 100644 crates/liquidity-sources/src/balancer_v2/pool_indexer.rs delete mode 100644 crates/liquidity-sources/src/balancer_v2/pool_init.rs delete mode 100644 crates/liquidity-sources/src/subgraph.rs diff --git a/crates/driver/example.toml b/crates/driver/example.toml index aeb4580ea1..397cd13a28 100644 --- a/crates/driver/example.toml +++ b/crates/driver/example.toml @@ -75,7 +75,7 @@ max-order-age = "1m" # [[liquidity.balancer-v2]] # Balancer V2 configuration # preset = "balancer-v2" -# graph-url = "http://localhost:1234" # which subgraph url to fetch the data from +# indexer-url = "http://localhost:7777" # pool-indexer base URL to fetch pool data from # pool-deny-list = [] # optional # [[liquidity.balancer-v2]] # Custom Balancer V2 configuration @@ -83,6 +83,7 @@ max-order-age = "1m" # weighted = [] # weighted pool factory addresses # stable = [] # stable pool factory addresses # liquidity-bootstrapping = [] # liquidity bootstrapping pool factory addresses +# indexer-url = "http://localhost:7777" # pool-indexer base URL to fetch pool data from # pool-deny-list = [] # which pools to ignore # [[liquidity.uniswap-v3]] # Uniswap V3 configuration (pool-indexer as data source) diff --git a/crates/driver/src/boundary/liquidity/balancer/v2/mod.rs b/crates/driver/src/boundary/liquidity/balancer/v2/mod.rs index 350a687555..6d61a96397 100644 --- a/crates/driver/src/boundary/liquidity/balancer/v2/mod.rs +++ b/crates/driver/src/boundary/liquidity/balancer/v2/mod.rs @@ -22,6 +22,7 @@ use { liquidity_sources::balancer_v2::{ BalancerPoolFetcher, pool_fetching::{BalancerContracts, BalancerFactoryInstance}, + pool_indexer::BalancerIndexerClient, }, shared::http_solver::model::TokenAmount, solver::{ @@ -171,12 +172,15 @@ async fn init_liquidity( let balancer_pool_fetcher = Arc::new( BalancerPoolFetcher::new( - &config.graph_url, + Box::new(BalancerIndexerClient::new( + config.indexer_url.clone(), + eth.chain(), + boundary::liquidity::http_client(), + )), block_retriever.clone(), token_info_fetcher.clone(), boundary::liquidity::cache_config(), block_stream.clone(), - boundary::liquidity::http_client(), web3.clone(), &contracts, config.pool_deny_list.to_vec(), diff --git a/crates/driver/src/infra/config/file/load.rs b/crates/driver/src/infra/config/file/load.rs index 4b53756e4e..5f096d2fda 100644 --- a/crates/driver/src/infra/config/file/load.rs +++ b/crates/driver/src/infra/config/file/load.rs @@ -265,14 +265,14 @@ pub async fn load(chain: Chain, path: &Path) -> infra::Config { file::BalancerV2Config::Preset { preset, pool_deny_list, - graph_url, + indexer_url, reinit_interval, } => liquidity::config::BalancerV2 { pool_deny_list: pool_deny_list.clone(), reinit_interval, ..match preset { file::BalancerV2Preset::BalancerV2 => { - liquidity::config::BalancerV2::balancer_v2(&graph_url, chain) + liquidity::config::BalancerV2::balancer_v2(&indexer_url, chain) } } .expect("no Balancer V2 preset for current network") @@ -285,7 +285,7 @@ pub async fn load(chain: Chain, path: &Path) -> infra::Config { liquidity_bootstrapping, composable_stable, pool_deny_list, - graph_url, + indexer_url, reinit_interval, } => liquidity::config::BalancerV2 { vault: vault.into(), @@ -295,7 +295,7 @@ pub async fn load(chain: Chain, path: &Path) -> infra::Config { liquidity_bootstrapping, composable_stable, pool_deny_list: pool_deny_list.clone(), - graph_url, + indexer_url, reinit_interval, }, }) diff --git a/crates/driver/src/infra/config/file/mod.rs b/crates/driver/src/infra/config/file/mod.rs index fdd3f9502a..006a48ce09 100644 --- a/crates/driver/src/infra/config/file/mod.rs +++ b/crates/driver/src/infra/config/file/mod.rs @@ -636,8 +636,8 @@ enum BalancerV2Config { #[serde(default)] pool_deny_list: Vec, - /// The URL used to connect to balancer v2 subgraph client. - graph_url: Url, + /// Base URL of the pool-indexer that seeds the pool registry. + indexer_url: Url, /// How often the liquidity source should be reinitialized to get /// access to new pools. @@ -677,8 +677,8 @@ enum BalancerV2Config { #[serde(default)] pool_deny_list: Vec, - /// The URL used to connect to balancer v2 subgraph client. - graph_url: Url, + /// Base URL of the pool-indexer that seeds the pool registry. + indexer_url: Url, /// How often the liquidity source should be reinitialized to get /// access to new pools. diff --git a/crates/driver/src/infra/liquidity/config.rs b/crates/driver/src/infra/liquidity/config.rs index fd48020c2b..260c5d60d5 100644 --- a/crates/driver/src/infra/liquidity/config.rs +++ b/crates/driver/src/infra/liquidity/config.rs @@ -224,8 +224,8 @@ pub struct BalancerV2 { /// ignored. pub pool_deny_list: Vec, - /// The base URL used to connect to balancer v2 subgraph client. - pub graph_url: Url, + /// Base URL of the pool-indexer that seeds the pool registry. + pub indexer_url: Url, /// How often the liquidty source should be re-initialized to become /// aware of new pools. @@ -235,7 +235,7 @@ pub struct BalancerV2 { impl BalancerV2 { /// Returns the liquidity configuration for Balancer V2. #[expect(clippy::self_named_constructors)] - pub fn balancer_v2(graph_url: &Url, chain: Chain) -> Option { + pub fn balancer_v2(indexer_url: &Url, chain: Chain) -> Option { macro_rules! address_for { ( $chain:expr, [ $( $($p:ident)::+ ),* $(,)? ] ) => {{ let arr = [ $({ @@ -282,7 +282,7 @@ impl BalancerV2 { ] ), pool_deny_list: Vec::new(), - graph_url: graph_url.clone(), + indexer_url: indexer_url.clone(), reinit_interval: None, }) } diff --git a/crates/liquidity-sources/src/balancer_v2/graph_api.rs b/crates/liquidity-sources/src/balancer_v2/graph_api.rs deleted file mode 100644 index 149338ad47..0000000000 --- a/crates/liquidity-sources/src/balancer_v2/graph_api.rs +++ /dev/null @@ -1,481 +0,0 @@ -//! Module containing The Graph API client used for retrieving Balancer weighted -//! pools from the Balancer V2 subgraph. -//! -//! The pools retrieved from this client are used to prime the graph event store -//! to reduce start-up time. We do not use this in general for retrieving pools -//! as to: -//! - not rely on external services -//! - ensure that we are using the latest up-to-date pool data by using events -//! from the node - -use { - super::swap::fixed_point::Bfp, - crate::{json_map, subgraph::SubgraphClient}, - alloy::primitives::{Address, B256}, - anyhow::Result, - event_indexing::event_handler::MAX_REORG_BLOCK_COUNT, - reqwest::{Client, Url}, - serde::Deserialize, - serde_json::json, - serde_with::{DisplayFromStr, serde_as}, - std::collections::HashMap, -}; - -/// The page size when querying pools. -#[cfg(not(test))] -const QUERY_PAGE_SIZE: usize = 1000; -#[cfg(test)] -const QUERY_PAGE_SIZE: usize = 10; - -/// A client to the Balancer V2 subgraph. -/// -/// This client is not implemented to allow general GraphQL queries, but instead -/// implements high-level methods that perform GraphQL queries under the hood. -pub struct BalancerSubgraphClient(SubgraphClient); - -impl BalancerSubgraphClient { - /// Creates a new Balancer subgraph client with full subgraph URL. - pub fn from_subgraph_url(subgraph_url: &Url, client: Client) -> Result { - Ok(Self(SubgraphClient::try_new(subgraph_url.clone(), client)?)) - } - - /// Retrieves the list of registered pools from the subgraph. - pub async fn get_registered_pools(&self) -> Result { - use self::pools_query::*; - - let block_number = self.get_safe_block().await?; - - let mut pools = Vec::new(); - let mut last_id = B256::default(); - - // We do paging by last ID instead of using `skip`. This is the - // suggested approach to paging best performance: - // - loop { - let page = self - .0 - .query::( - QUERY, - Some(json_map! { - "block" => block_number, - "pageSize" => QUERY_PAGE_SIZE, - "lastId" => json!(last_id), - }), - ) - .await? - .pools; - let no_more_pages = page.len() != QUERY_PAGE_SIZE; - if let Some(last_pool) = page.last() { - last_id = last_pool.id; - } - - pools.extend(page); - - if no_more_pages { - break; - } - } - - Ok(RegisteredPools { - fetched_block_number: block_number, - pools, - }) - } - - /// Retrieves a recent block number for which it is safe to assume no - /// reorgs will happen. - async fn get_safe_block(&self) -> Result { - // Ideally we would want to use block hash here so that we can check - // that there indeed is no reorg. However, it does not seem possible to - // retrieve historic block hashes just from the subgraph (it always - // returns `null`). - Ok(self - .0 - .query::(block_number_query::QUERY, None) - .await? - .meta - .block - .number - .saturating_sub(MAX_REORG_BLOCK_COUNT)) - } -} - -/// Result of the registered stable pool query. -#[derive(Debug, Default, Eq, PartialEq)] -pub struct RegisteredPools { - /// The block number that the data was fetched, and for which the registered - /// weighted pools can be considered up to date. - pub fetched_block_number: u64, - /// The registered Pools - pub pools: Vec, -} - -impl RegisteredPools { - /// Creates an empty collection of registered pools for the specified block - /// number. - pub fn empty(fetched_block_number: u64) -> Self { - Self { - fetched_block_number, - ..Default::default() - } - } - - /// Groups registered pools by factory addresses. - pub fn group_by_factory(self) -> HashMap { - let fetched_block_number = self.fetched_block_number; - self.pools - .into_iter() - .fold(HashMap::new(), |mut grouped, pool| { - grouped - .entry(pool.factory) - .or_insert(RegisteredPools { - fetched_block_number, - ..Default::default() - }) - .pools - .push(pool); - grouped - }) - } -} - -/// Pool data from the Balancer V2 subgraph. -#[derive(Debug, Deserialize, Eq, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct PoolData { - pub pool_type: PoolType, - pub id: B256, - pub address: Address, - pub factory: Address, - pub swap_enabled: bool, - pub tokens: Vec, -} - -/// Supported pool kinds. -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Hash)] -pub enum PoolType { - Stable, - Weighted, - LiquidityBootstrapping, - ComposableStable, -} - -/// Token data for pools. -#[serde_as] -#[derive(Debug, Deserialize, Eq, PartialEq)] -pub struct Token { - pub address: Address, - pub decimals: u8, - #[serde_as(as = "Option")] - #[serde(default)] - pub weight: Option, -} - -mod pools_query { - use {super::PoolData, serde::Deserialize}; - - pub const QUERY: &str = r#" - query Pools($block: Int, $pageSize: Int, $lastId: ID) { - pools( - block: { number: $block } - first: $pageSize - where: { - id_gt: $lastId - poolType_in: [ - "Stable", - "Weighted", - "LiquidityBootstrapping", - "ComposableStable", - ] - totalLiquidity_gt: "1" # 1$ value of tokens - } - ) { - poolType - id - address - factory - swapEnabled - tokens { - address - decimals - weight - } - } - } - "#; - - #[derive(Debug, Deserialize, Eq, PartialEq)] - pub struct Data { - pub pools: Vec, - } -} - -mod block_number_query { - use serde::Deserialize; - - pub const QUERY: &str = r#"{ - _meta { - block { number } - } - }"#; - - #[derive(Debug, Deserialize, Eq, PartialEq)] - pub struct Data { - #[serde(rename = "_meta")] - pub meta: Meta, - } - - #[derive(Debug, Deserialize, Eq, PartialEq)] - pub struct Meta { - pub block: Block, - } - - #[derive(Debug, Deserialize, Eq, PartialEq)] - pub struct Block { - pub number: u64, - } -} - -#[cfg(test)] -mod tests { - use { - super::*, - crate::balancer_v2::swap::fixed_point::Bfp, - alloy::primitives::U256, - maplit::hashmap, - }; - - #[test] - fn decode_pools_data() { - use pools_query::*; - - assert_eq!( - serde_json::from_value::(json!({ - "pools": [ - { - "poolType": "Weighted", - "address": "0x2222222222222222222222222222222222222222", - "id": "0x1111111111111111111111111111111111111111111111111111111111111111", - "factory": "0x5555555555555555555555555555555555555555", - "swapEnabled": true, - "tokens": [ - { - "address": "0x3333333333333333333333333333333333333333", - "decimals": 3, - "weight": "0.5" - }, - { - "address": "0x4444444444444444444444444444444444444444", - "decimals": 4, - "weight": "0.5" - }, - ], - }, - { - "poolType": "Stable", - "address": "0x2222222222222222222222222222222222222222", - "id": "0x1111111111111111111111111111111111111111111111111111111111111111", - "factory": "0x5555555555555555555555555555555555555555", - "swapEnabled": true, - "tokens": [ - { - "address": "0x3333333333333333333333333333333333333333", - "decimals": 3, - }, - { - "address": "0x4444444444444444444444444444444444444444", - "decimals": 4, - }, - ], - }, - { - "poolType": "LiquidityBootstrapping", - "address": "0x2222222222222222222222222222222222222222", - "id": "0x1111111111111111111111111111111111111111111111111111111111111111", - "factory": "0x5555555555555555555555555555555555555555", - "swapEnabled": true, - "tokens": [ - { - "address": "0x3333333333333333333333333333333333333333", - "decimals": 3, - "weight": "0.5" - }, - { - "address": "0x4444444444444444444444444444444444444444", - "decimals": 4, - "weight": "0.5" - }, - ], - }, - { - "poolType": "ComposableStable", - "address": "0x2222222222222222222222222222222222222222", - "id": "0x1111111111111111111111111111111111111111111111111111111111111111", - "factory": "0x5555555555555555555555555555555555555555", - "swapEnabled": true, - "tokens": [ - { - "address": "0x3333333333333333333333333333333333333333", - "decimals": 3, - }, - { - "address": "0x4444444444444444444444444444444444444444", - "decimals": 4, - }, - ], - }, - ], - })) - .unwrap(), - Data { - pools: vec![ - PoolData { - pool_type: PoolType::Weighted, - id: B256::repeat_byte(0x11), - address: Address::repeat_byte(0x22), - factory: Address::repeat_byte(0x55), - swap_enabled: true, - tokens: vec![ - Token { - address: Address::repeat_byte(0x33), - decimals: 3, - weight: Some(Bfp::from_wei(U256::from( - 500_000_000_000_000_000_u128 - ))), - }, - Token { - address: Address::repeat_byte(0x44), - decimals: 4, - weight: Some(Bfp::from_wei(U256::from( - 500_000_000_000_000_000_u128 - ))), - }, - ], - }, - PoolData { - pool_type: PoolType::Stable, - id: B256::repeat_byte(0x11), - address: Address::repeat_byte(0x22), - factory: Address::repeat_byte(0x55), - swap_enabled: true, - tokens: vec![ - Token { - address: Address::repeat_byte(0x33), - decimals: 3, - weight: None, - }, - Token { - address: Address::repeat_byte(0x44), - decimals: 4, - weight: None, - }, - ], - }, - PoolData { - pool_type: PoolType::LiquidityBootstrapping, - id: B256::repeat_byte(0x11), - address: Address::repeat_byte(0x22), - factory: Address::repeat_byte(0x55), - swap_enabled: true, - tokens: vec![ - Token { - address: Address::repeat_byte(0x33), - decimals: 3, - weight: Some(Bfp::from_wei(U256::from( - 500_000_000_000_000_000_u128 - ))), - }, - Token { - address: Address::repeat_byte(0x44), - decimals: 4, - weight: Some(Bfp::from_wei(U256::from( - 500_000_000_000_000_000_u128 - ))), - }, - ], - }, - PoolData { - pool_type: PoolType::ComposableStable, - id: B256::repeat_byte(0x11), - address: Address::repeat_byte(0x22), - factory: Address::repeat_byte(0x55), - swap_enabled: true, - tokens: vec![ - Token { - address: Address::repeat_byte(0x33), - decimals: 3, - weight: None, - }, - Token { - address: Address::repeat_byte(0x44), - decimals: 4, - weight: None, - }, - ], - }, - ], - } - ); - } - - #[test] - fn decode_block_number_data() { - use block_number_query::*; - - assert_eq!( - serde_json::from_value::(json!({ - "_meta": { - "block": { - "number": 42, - }, - }, - })) - .unwrap(), - Data { - meta: Meta { - block: Block { number: 42 } - } - } - ); - } - - #[test] - fn groups_pools_by_factory() { - let pool = |factory: Address, id: u8| PoolData { - id: B256::repeat_byte(id), - factory, - pool_type: PoolType::Weighted, - address: Default::default(), - swap_enabled: true, - tokens: Default::default(), - }; - - let registered_pools = RegisteredPools { - pools: vec![ - pool(Address::repeat_byte(1), 1), - pool(Address::repeat_byte(1), 2), - pool(Address::repeat_byte(2), 3), - ], - fetched_block_number: 42, - }; - - assert_eq!( - registered_pools.group_by_factory(), - hashmap! { - Address::repeat_byte(1) => RegisteredPools { - pools: vec![ - pool(Address::repeat_byte(1), 1), - pool(Address::repeat_byte(1), 2), - ], - fetched_block_number: 42, - }, - Address::repeat_byte(2) => RegisteredPools { - pools: vec![ - pool(Address::repeat_byte(2), 3), - ], - fetched_block_number: 42, - }, - } - ) - } -} diff --git a/crates/liquidity-sources/src/balancer_v2/mod.rs b/crates/liquidity-sources/src/balancer_v2/mod.rs index b9b4a9de5c..dcbdd8610b 100644 --- a/crates/liquidity-sources/src/balancer_v2/mod.rs +++ b/crates/liquidity-sources/src/balancer_v2/mod.rs @@ -30,20 +30,29 @@ //! respectively along with the current balances of each of the pool's tokens //! (aka the pool's "reserves"). //! -//! For this reason, only the `event_handler`, `pool_cache`, `pool_fetching` and -//! `swap` are declared as public, others merely contain internal logic -//! regarding how information is collected and stored. +//! For this reason, only the `models`, `pool_fetching`, `pool_indexer`, `pools` +//! and `swap` modules are declared as public; others merely contain internal +//! logic regarding how information is collected and stored. //! //! Once should think of `PoolStorage` as a type of Database for which one is //! not concerned with how it maintains itself. -mod graph_api; +pub mod models; pub mod pool_fetching; -mod pool_init; +pub mod pool_indexer; pub mod pools; pub mod swap; +use {anyhow::Result, models::RegisteredPools}; + pub use self::{ pool_fetching::{BalancerPoolFetcher, BalancerPoolFetching}, pools::{Pool, PoolKind}, }; + +/// Seeds the balancer pool registry at start-up with the currently registered +/// pools. Implemented by [`pool_indexer::BalancerIndexerClient`]. +#[async_trait::async_trait] +pub trait PoolInitializing: Send + Sync + 'static { + async fn initialize_pools(&self) -> Result; +} diff --git a/crates/liquidity-sources/src/balancer_v2/models.rs b/crates/liquidity-sources/src/balancer_v2/models.rs new file mode 100644 index 0000000000..868a62b126 --- /dev/null +++ b/crates/liquidity-sources/src/balancer_v2/models.rs @@ -0,0 +1,283 @@ +//! Static Balancer V2 pool data — the registry seed consumed by the aggregate +//! pool fetcher. Populated by the pool-indexer client. + +use { + super::swap::fixed_point::Bfp, + alloy::primitives::{Address, B256}, + serde::Deserialize, + serde_with::{DisplayFromStr, serde_as}, + std::collections::HashMap, +}; + +/// A set of registered pools, up to date as of `fetched_block_number`. +#[derive(Debug, Default, Eq, PartialEq)] +pub struct RegisteredPools { + /// The block the pools were fetched for and can be considered current at. + pub fetched_block_number: u64, + /// The registered pools. + pub pools: Vec, +} + +impl RegisteredPools { + /// Creates an empty collection for the specified block number. + pub fn empty(fetched_block_number: u64) -> Self { + Self { + fetched_block_number, + ..Default::default() + } + } + + /// Groups registered pools by factory address. + pub fn group_by_factory(self) -> HashMap { + let fetched_block_number = self.fetched_block_number; + self.pools + .into_iter() + .fold(HashMap::new(), |mut grouped, pool| { + grouped + .entry(pool.factory) + .or_insert(RegisteredPools { + fetched_block_number, + ..Default::default() + }) + .pools + .push(pool); + grouped + }) + } +} + +/// Static data for a Balancer V2 pool. +#[derive(Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct PoolData { + pub pool_type: PoolType, + pub id: B256, + pub address: Address, + pub factory: Address, + pub swap_enabled: bool, + pub tokens: Vec, +} + +/// Supported pool kinds. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Hash)] +pub enum PoolType { + Stable, + Weighted, + LiquidityBootstrapping, + ComposableStable, +} + +/// Token data for pools. `weight` is present only for weighted pools. +#[serde_as] +#[derive(Debug, Deserialize, Eq, PartialEq)] +pub struct Token { + pub address: Address, + pub decimals: u8, + #[serde_as(as = "Option")] + #[serde(default)] + pub weight: Option, +} + +#[cfg(test)] +mod tests { + use {super::*, alloy::primitives::U256, maplit::hashmap, serde_json::json}; + + #[test] + fn decode_pools_data() { + assert_eq!( + serde_json::from_value::>(json!([ + { + "poolType": "Weighted", + "address": "0x2222222222222222222222222222222222222222", + "id": "0x1111111111111111111111111111111111111111111111111111111111111111", + "factory": "0x5555555555555555555555555555555555555555", + "swapEnabled": true, + "tokens": [ + { + "address": "0x3333333333333333333333333333333333333333", + "decimals": 3, + "weight": "0.5" + }, + { + "address": "0x4444444444444444444444444444444444444444", + "decimals": 4, + "weight": "0.5" + }, + ], + }, + { + "poolType": "Stable", + "address": "0x2222222222222222222222222222222222222222", + "id": "0x1111111111111111111111111111111111111111111111111111111111111111", + "factory": "0x5555555555555555555555555555555555555555", + "swapEnabled": true, + "tokens": [ + { + "address": "0x3333333333333333333333333333333333333333", + "decimals": 3, + }, + { + "address": "0x4444444444444444444444444444444444444444", + "decimals": 4, + }, + ], + }, + { + "poolType": "LiquidityBootstrapping", + "address": "0x2222222222222222222222222222222222222222", + "id": "0x1111111111111111111111111111111111111111111111111111111111111111", + "factory": "0x5555555555555555555555555555555555555555", + "swapEnabled": true, + "tokens": [ + { + "address": "0x3333333333333333333333333333333333333333", + "decimals": 3, + "weight": "0.5" + }, + { + "address": "0x4444444444444444444444444444444444444444", + "decimals": 4, + "weight": "0.5" + }, + ], + }, + { + "poolType": "ComposableStable", + "address": "0x2222222222222222222222222222222222222222", + "id": "0x1111111111111111111111111111111111111111111111111111111111111111", + "factory": "0x5555555555555555555555555555555555555555", + "swapEnabled": true, + "tokens": [ + { + "address": "0x3333333333333333333333333333333333333333", + "decimals": 3, + }, + { + "address": "0x4444444444444444444444444444444444444444", + "decimals": 4, + }, + ], + }, + ])) + .unwrap(), + vec![ + PoolData { + pool_type: PoolType::Weighted, + id: B256::repeat_byte(0x11), + address: Address::repeat_byte(0x22), + factory: Address::repeat_byte(0x55), + swap_enabled: true, + tokens: vec![ + Token { + address: Address::repeat_byte(0x33), + decimals: 3, + weight: Some(Bfp::from_wei(U256::from(500_000_000_000_000_000_u128))), + }, + Token { + address: Address::repeat_byte(0x44), + decimals: 4, + weight: Some(Bfp::from_wei(U256::from(500_000_000_000_000_000_u128))), + }, + ], + }, + PoolData { + pool_type: PoolType::Stable, + id: B256::repeat_byte(0x11), + address: Address::repeat_byte(0x22), + factory: Address::repeat_byte(0x55), + swap_enabled: true, + tokens: vec![ + Token { + address: Address::repeat_byte(0x33), + decimals: 3, + weight: None, + }, + Token { + address: Address::repeat_byte(0x44), + decimals: 4, + weight: None, + }, + ], + }, + PoolData { + pool_type: PoolType::LiquidityBootstrapping, + id: B256::repeat_byte(0x11), + address: Address::repeat_byte(0x22), + factory: Address::repeat_byte(0x55), + swap_enabled: true, + tokens: vec![ + Token { + address: Address::repeat_byte(0x33), + decimals: 3, + weight: Some(Bfp::from_wei(U256::from(500_000_000_000_000_000_u128))), + }, + Token { + address: Address::repeat_byte(0x44), + decimals: 4, + weight: Some(Bfp::from_wei(U256::from(500_000_000_000_000_000_u128))), + }, + ], + }, + PoolData { + pool_type: PoolType::ComposableStable, + id: B256::repeat_byte(0x11), + address: Address::repeat_byte(0x22), + factory: Address::repeat_byte(0x55), + swap_enabled: true, + tokens: vec![ + Token { + address: Address::repeat_byte(0x33), + decimals: 3, + weight: None, + }, + Token { + address: Address::repeat_byte(0x44), + decimals: 4, + weight: None, + }, + ], + }, + ] + ); + } + + #[test] + fn groups_pools_by_factory() { + let pool = |factory: Address, id: u8| PoolData { + id: B256::repeat_byte(id), + factory, + pool_type: PoolType::Weighted, + address: Default::default(), + swap_enabled: true, + tokens: Default::default(), + }; + + let registered_pools = RegisteredPools { + pools: vec![ + pool(Address::repeat_byte(1), 1), + pool(Address::repeat_byte(1), 2), + pool(Address::repeat_byte(2), 3), + ], + fetched_block_number: 42, + }; + + assert_eq!( + registered_pools.group_by_factory(), + hashmap! { + Address::repeat_byte(1) => RegisteredPools { + pools: vec![ + pool(Address::repeat_byte(1), 1), + pool(Address::repeat_byte(1), 2), + ], + fetched_block_number: 42, + }, + Address::repeat_byte(2) => RegisteredPools { + pools: vec![ + pool(Address::repeat_byte(2), 3), + ], + fetched_block_number: 42, + }, + } + ) + } +} diff --git a/crates/liquidity-sources/src/balancer_v2/pool_fetching/mod.rs b/crates/liquidity-sources/src/balancer_v2/pool_fetching/mod.rs index 1dfdaab5d2..d0ada41630 100644 --- a/crates/liquidity-sources/src/balancer_v2/pool_fetching/mod.rs +++ b/crates/liquidity-sources/src/balancer_v2/pool_fetching/mod.rs @@ -12,8 +12,8 @@ use { registry::Registry, }, super::{ - graph_api::{BalancerSubgraphClient, RegisteredPools}, - pool_init::PoolInitializing, + PoolInitializing, + models::RegisteredPools, pools::{ FactoryIndexing, Pool, @@ -50,7 +50,6 @@ use { ethrpc::{Web3, alloy::ProviderLabelingExt, block_stream::CurrentBlockWatcher}, event_indexing::block_retriever::BlockRetrieving, model::TokenPair, - reqwest::{Client, Url}, std::{ collections::{BTreeMap, HashSet}, sync::Arc, @@ -234,17 +233,15 @@ pub struct BalancerContracts { impl BalancerPoolFetcher { #[expect(clippy::too_many_arguments)] pub async fn new( - subgraph_url: &Url, + pool_initializer: Box, block_retriever: Arc, token_infos: Arc, config: CacheConfig, block_stream: CurrentBlockWatcher, - client: Client, web3: Web3, contracts: &BalancerContracts, deny_listed_pool_ids: Vec, ) -> Result { - let pool_initializer = BalancerSubgraphClient::from_subgraph_url(subgraph_url, client)?; let web3 = web3.labeled("balancerV2"); let fetcher = Arc::new(Cache::new( create_aggregate_pool_fetcher( @@ -316,7 +313,7 @@ impl BalancerPoolFetching for BalancerPoolFetcher { /// Creates an aggregate fetcher for all supported pool factories. async fn create_aggregate_pool_fetcher( web3: Web3, - pool_initializer: impl PoolInitializing, + pool_initializer: Box, block_retriever: Arc, token_infos: Arc, contracts: &BalancerContracts, diff --git a/crates/liquidity-sources/src/balancer_v2/pool_indexer.rs b/crates/liquidity-sources/src/balancer_v2/pool_indexer.rs new file mode 100644 index 0000000000..a37e69bb7f --- /dev/null +++ b/crates/liquidity-sources/src/balancer_v2/pool_indexer.rs @@ -0,0 +1,147 @@ +//! HTTP client for the Balancer V2 pools served by the pool-indexer service. +//! +//! Implements [`PoolInitializing`] so the pool-indexer can stand in for the +//! subgraph as the source that seeds the pool registry at start-up. The +//! indexer's `PoolData` response shares the subgraph's wire shape, so the +//! pages deserialize straight into [`PoolData`] and need no remapping. + +use { + super::{ + PoolInitializing, + models::{PoolData, RegisteredPools}, + }, + anyhow::{Context, Result}, + chain::Chain, + reqwest::{Client, Url}, + serde::Deserialize, + std::time::Duration, +}; + +/// Matches the server-side `MAX_PAGE_LIMIT`. +const LIST_PAGE_SIZE: u64 = 5000; + +/// Poll interval while the indexer is still bootstrapping (503). +const READY_POLL_INTERVAL: Duration = Duration::from_millis(500); + +/// Cap on the start-up wait for the indexer to serve its first checkpoint. +const READY_TIMEOUT: Duration = Duration::from_secs(60); + +pub struct BalancerIndexerClient { + base_url: Url, + http: Client, +} + +impl BalancerIndexerClient { + pub fn new(base_url: Url, chain: Chain, http: Client) -> Self { + let prefix = format!("api/v1/{}/balancer/v2/", chain.as_str()); + Self { + base_url: url_join(&base_url, &prefix), + http, + } + } + + fn path(&self, suffix: &str) -> Url { + url_join(&self.base_url, suffix) + } + + /// `GET /pools?limit=N[&after=cursor]`. `None` means the indexer replied + /// 503 — still bootstrapping, no checkpoint yet. + async fn fetch_pools_page( + &self, + limit: u64, + cursor: Option<&str>, + ) -> Result> { + let mut url = self.path("pools"); + url.query_pairs_mut() + .append_pair("limit", &limit.to_string()); + if let Some(c) = cursor { + url.query_pairs_mut().append_pair("after", c); + } + let resp = self.http.get(url).send().await.context("GET /pools")?; + if resp.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE { + return Ok(None); + } + let page = resp + .error_for_status() + .context("/pools HTTP status")? + .json() + .await + .context("/pools body")?; + Ok(Some(page)) + } + + /// Polls `/pools` until the indexer is past bootstrap (not 503), bounded by + /// [`READY_TIMEOUT`]. Covers the serve container coming up moments after + /// the driver; a cold bootstrap is expected to have run before then. + async fn wait_until_ready(&self) -> Result<()> { + let deadline = std::time::Instant::now() + READY_TIMEOUT; + loop { + if self.fetch_pools_page(1, None).await?.is_some() { + return Ok(()); + } + if std::time::Instant::now() >= deadline { + anyhow::bail!("balancer pool-indexer not ready after {READY_TIMEOUT:?}"); + } + tracing::debug!("balancer pool-indexer not ready yet (503); waiting"); + tokio::time::sleep(READY_POLL_INTERVAL).await; + } + } +} + +#[async_trait::async_trait] +impl PoolInitializing for BalancerIndexerClient { + async fn initialize_pools(&self) -> Result { + self.wait_until_ready().await?; + + let mut cursor: Option = None; + let mut pools: Vec = Vec::new(); + let mut fetched_block_number: Option = None; + loop { + let page = self + .fetch_pools_page(LIST_PAGE_SIZE, cursor.as_deref()) + .await? + .context("balancer pool-indexer returned 503 after readiness check")?; + fetched_block_number.get_or_insert(page.block_number); + pools.extend(page.pools); + match page.next_cursor { + Some(c) => cursor = Some(c), + None => break, + } + } + + let registered_pools = RegisteredPools { + fetched_block_number: fetched_block_number + .context("balancer pool-indexer returned no pages")?, + pools, + }; + tracing::debug!( + block = %registered_pools.fetched_block_number, + pools = %registered_pools.pools.len(), + "initialized registered pools from indexer", + ); + Ok(registered_pools) + } +} + +/// Wire form of a `/pools` page. `pools` reuses the subgraph's [`PoolData`]. +#[derive(Deserialize)] +struct PoolsResponse { + block_number: u64, + pools: Vec, + #[serde(default)] + next_cursor: Option, +} + +/// Joins `path` onto `url` with exactly one slash between them. `Url::join` +/// drops a base's last path segment when it lacks a trailing slash (RFC 3986 +/// path resolution), and the operator-supplied indexer URL may omit one. +fn url_join(url: &Url, mut path: &str) -> Url { + let mut url = url.to_string(); + while url.ends_with('/') { + url.pop(); + } + while path.starts_with('/') { + path = &path[1..]; + } + Url::parse(&format!("{url}/{path}")).expect("constructed URL is valid") +} diff --git a/crates/liquidity-sources/src/balancer_v2/pool_init.rs b/crates/liquidity-sources/src/balancer_v2/pool_init.rs deleted file mode 100644 index 29e349c27c..0000000000 --- a/crates/liquidity-sources/src/balancer_v2/pool_init.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! Balancer pool registry initialization. -//! -//! This module contains a component used to initialize Balancer pool registries -//! with existing data in order to reduce the "cold start" time of the service. - -use { - super::graph_api::{BalancerSubgraphClient, RegisteredPools}, - anyhow::Result, -}; - -#[async_trait::async_trait] -pub trait PoolInitializing: Send + Sync { - async fn initialize_pools(&self) -> Result; -} - -#[async_trait::async_trait] -impl PoolInitializing for BalancerSubgraphClient { - async fn initialize_pools(&self) -> Result { - let registered_pools = self.get_registered_pools().await?; - tracing::debug!( - block = %registered_pools.fetched_block_number, pools = %registered_pools.pools.len(), - "initialized registered pools", - ); - - Ok(registered_pools) - } -} diff --git a/crates/liquidity-sources/src/balancer_v2/pools/common.rs b/crates/liquidity-sources/src/balancer_v2/pools/common.rs index c9c6c831d3..e795915a6b 100644 --- a/crates/liquidity-sources/src/balancer_v2/pools/common.rs +++ b/crates/liquidity-sources/src/balancer_v2/pools/common.rs @@ -3,7 +3,7 @@ use { super::{FactoryIndexing, Pool, PoolIndexing as _, PoolStatus}, crate::balancer_v2::{ - graph_api::{PoolData, PoolType}, + models::{PoolData, PoolType}, swap::fixed_point::Bfp, }, alloy::{ @@ -357,7 +357,7 @@ mod tests { crate::{ balancer_v2::{ PoolKind, - graph_api::{PoolType, Token}, + models::{PoolType, Token}, pools::{MockFactoryIndexing, weighted}, }, bfp, diff --git a/crates/liquidity-sources/src/balancer_v2/pools/composable_stable.rs b/crates/liquidity-sources/src/balancer_v2/pools/composable_stable.rs index 5f5164fa37..5a10ece7de 100644 --- a/crates/liquidity-sources/src/balancer_v2/pools/composable_stable.rs +++ b/crates/liquidity-sources/src/balancer_v2/pools/composable_stable.rs @@ -3,7 +3,7 @@ use { super::{FactoryIndexing, PoolIndexing, common}, crate::balancer_v2::{ - graph_api::{PoolData, PoolType}, + models::{PoolData, PoolType}, swap::fixed_point::Bfp, }, alloy::eips::BlockId, @@ -112,7 +112,7 @@ impl FactoryIndexing for BalancerV2ComposableStablePoolFactory::Instance { mod tests { use { super::*, - crate::balancer_v2::graph_api::Token, + crate::balancer_v2::models::Token, alloy::primitives::{Address, B256}, }; diff --git a/crates/liquidity-sources/src/balancer_v2/pools/liquidity_bootstrapping.rs b/crates/liquidity-sources/src/balancer_v2/pools/liquidity_bootstrapping.rs index 535cfdfef2..b8a1b64f43 100644 --- a/crates/liquidity-sources/src/balancer_v2/pools/liquidity_bootstrapping.rs +++ b/crates/liquidity-sources/src/balancer_v2/pools/liquidity_bootstrapping.rs @@ -3,7 +3,7 @@ use { super::{FactoryIndexing, PoolIndexing, common}, crate::balancer_v2::{ - graph_api::{PoolData, PoolType}, + models::{PoolData, PoolType}, swap::fixed_point::Bfp, }, alloy::eips::BlockId, @@ -118,7 +118,7 @@ impl FactoryIndexing for BalancerV2LiquidityBootstrappingPoolFactory::Instance { mod tests { use { super::*, - crate::balancer_v2::graph_api::Token, + crate::balancer_v2::models::Token, alloy::primitives::{Address, B256}, }; diff --git a/crates/liquidity-sources/src/balancer_v2/pools/mod.rs b/crates/liquidity-sources/src/balancer_v2/pools/mod.rs index 2d3138bd85..d757999c12 100644 --- a/crates/liquidity-sources/src/balancer_v2/pools/mod.rs +++ b/crates/liquidity-sources/src/balancer_v2/pools/mod.rs @@ -14,7 +14,7 @@ pub mod stable; pub mod weighted; use { - super::graph_api::PoolData, + super::models::PoolData, alloy::{eips::BlockId, primitives::B256}, anyhow::Result, futures::future::BoxFuture, diff --git a/crates/liquidity-sources/src/balancer_v2/pools/stable.rs b/crates/liquidity-sources/src/balancer_v2/pools/stable.rs index 21ad5dd031..060fa309ce 100644 --- a/crates/liquidity-sources/src/balancer_v2/pools/stable.rs +++ b/crates/liquidity-sources/src/balancer_v2/pools/stable.rs @@ -3,7 +3,7 @@ use { super::{FactoryIndexing, PoolIndexing, common}, crate::balancer_v2::{ - graph_api::{PoolData, PoolType}, + models::{PoolData, PoolType}, swap::fixed_point::Bfp, }, alloy::{ @@ -127,7 +127,7 @@ impl FactoryIndexing for BalancerV2StablePoolFactoryV2::Instance { #[cfg(test)] mod tests { - use {super::*, crate::balancer_v2::graph_api::Token, alloy::primitives::B256}; + use {super::*, crate::balancer_v2::models::Token, alloy::primitives::B256}; #[test] fn errors_when_converting_wrong_pool_type() { diff --git a/crates/liquidity-sources/src/balancer_v2/pools/weighted.rs b/crates/liquidity-sources/src/balancer_v2/pools/weighted.rs index 2e654de6a5..ac10de96df 100644 --- a/crates/liquidity-sources/src/balancer_v2/pools/weighted.rs +++ b/crates/liquidity-sources/src/balancer_v2/pools/weighted.rs @@ -3,7 +3,7 @@ use { super::{FactoryIndexing, PoolIndexing, common}, crate::balancer_v2::{ - graph_api::{PoolData, PoolType}, + models::{PoolData, PoolType}, swap::fixed_point::Bfp, }, alloy::{eips::BlockId, primitives::Address}, @@ -145,7 +145,7 @@ fn pool_state( mod tests { use { super::*, - crate::{balancer_v2::graph_api::Token, bfp}, + crate::{balancer_v2::models::Token, bfp}, alloy::{ primitives::{Address, B256, U256}, providers::{Provider, ProviderBuilder, mock::Asserter}, diff --git a/crates/liquidity-sources/src/lib.rs b/crates/liquidity-sources/src/lib.rs index d6c106ff32..002dc40062 100644 --- a/crates/liquidity-sources/src/lib.rs +++ b/crates/liquidity-sources/src/lib.rs @@ -5,7 +5,6 @@ pub mod base_tokens; pub mod baseline_solvable; mod macros; pub mod recent_block_cache; -pub mod subgraph; pub mod swapr; pub mod uniswap_v2; pub mod uniswap_v3; diff --git a/crates/liquidity-sources/src/macros.rs b/crates/liquidity-sources/src/macros.rs index c9fbf51f3a..a0ceb8e365 100644 --- a/crates/liquidity-sources/src/macros.rs +++ b/crates/liquidity-sources/src/macros.rs @@ -6,15 +6,3 @@ macro_rules! bfp { .unwrap() }; } - -#[macro_export] -macro_rules! json_map { - ($($key:expr_2021 => $value:expr_2021),* $(,)?) => {{ - #[allow(unused_mut)] - let mut map = ::serde_json::Map::::new(); - $( - map.insert(($key).into(), ($value).into()); - )* - map - }} -} diff --git a/crates/liquidity-sources/src/subgraph.rs b/crates/liquidity-sources/src/subgraph.rs deleted file mode 100644 index 61fed055f6..0000000000 --- a/crates/liquidity-sources/src/subgraph.rs +++ /dev/null @@ -1,303 +0,0 @@ -//! A module implementing a client for querying subgraphs. - -use { - crate::json_map, - anyhow::{Result, bail}, - reqwest::{Client, Url}, - serde::{Deserialize, Serialize, de::DeserializeOwned}, - serde_json::{Map, Value, json}, - thiserror::Error, -}; - -pub const QUERY_PAGE_SIZE: usize = 1000; -const MAX_NUMBER_OF_ATTEMPTS_DEFAULT: usize = 10; - -/// A general client for querying subgraphs. -pub struct SubgraphClient { - client: Client, - subgraph_url: Url, - max_number_of_attempts: usize, -} - -pub trait ContainsId { - fn get_id(&self) -> String; -} - -#[derive(Debug, Deserialize, Eq, PartialEq)] -pub struct Data { - #[serde(alias = "pools", alias = "ticks")] - pub inner: Vec, -} - -impl SubgraphClient { - /// Creates a new subgraph client from the specified organization and name. - pub fn try_new(subgraph_url: Url, client: Client) -> Result { - Ok(Self { - client, - subgraph_url, - max_number_of_attempts: MAX_NUMBER_OF_ATTEMPTS_DEFAULT, - }) - } - - /// Performs the specified GraphQL query on the current subgraph. - pub async fn query(&self, query: &str, variables: Option>) -> Result - where - T: DeserializeOwned, - { - // for long lasting queries subgraph call might randomly fail - // introduced retry mechanism that should efficiently help since failures are - // quick and we need 1 or 2 retries to succeed. - let mut error: Option = None; - for _ in 0..self.max_number_of_attempts { - match self.query_without_retry(query, &variables).await { - Ok(result) => return Ok(result), - Err(err) => error = Some(err), - } - } - Err(anyhow::anyhow!(format!( - "failed to execute query on subgraph: {}", - error.unwrap() - ))) - } - - pub async fn query_without_retry( - &self, - query: &str, - variables: &Option>, - ) -> Result - where - T: DeserializeOwned, - { - match self - .client - .post(self.subgraph_url.clone()) - .json(&Query { - query, - variables: variables.clone(), - }) - .send() - .await? - .json::>() - .await? - .into_result() - { - Ok(result) => Ok(result), - Err(err) => { - tracing::warn!("failed to query subgraph: {}", err); - Err(anyhow::anyhow!(format!( - "failed to execute query on subgraph: {}", - err - ))) - } - } - } - - /// Performs the specified GraphQL query on the current subgraph. - /// This function should be called for queries that return very - /// long(paginated) result. - pub async fn paginated_query( - &self, - query: &str, - mut variables: Map, - ) -> Result> - where - T: ContainsId + DeserializeOwned, - { - let mut result = Vec::new(); - - // We do paging by last ID instead of using `skip`. This is the - // suggested approach to paging best performance: - // - variables.extend(json_map! { - "pageSize" => QUERY_PAGE_SIZE, - "lastId" => json!(String::default()), - }); - loop { - let page = self - .query::>(query, Some(variables.clone())) - .await? - .inner; - let no_more_pages = page.len() != QUERY_PAGE_SIZE; - if let Some(last_elem) = page.last() { - variables.insert("lastId".to_string(), json!(last_elem.get_id())); - } - - result.extend(page); - - if no_more_pages { - break; - } - } - - Ok(result) - } -} - -/// A GraphQL query. -#[derive(Serialize)] -struct Query<'a> { - query: &'a str, - variables: Option>, -} - -/// A GraphQL query response. -/// -/// This type gets converted into a Rust `Result` type, while handling invalid -/// responses (with missing data and errors). -#[derive(Debug, Deserialize)] -struct QueryResponse { - #[serde(default = "empty_data")] - data: Option, - #[serde(default)] - errors: Option>, -} - -impl QueryResponse { - fn into_result(self) -> Result { - match self { - Self { - data: Some(data), - errors: None, - } => Ok(data), - Self { - errors: Some(errors), - data: None, - } if !errors.is_empty() => { - // Make sure to log additional errors if there are more than - // one, and just bubble up the first error. - for error in &errors[1..] { - tracing::warn!("additional GraphQL error: {}", error.message); - } - bail!("{}", errors[0]) - } - _ => bail!("invalid GraphQL response"), - } - } -} - -#[derive(Debug, Deserialize, Error)] -#[error("{}", .message)] -struct QueryError { - message: String, -} - -/// Function to work around the fact that `#[serde(default)]` on an `Option` -/// requires `T: Default`. -fn empty_data() -> Option { - None -} - -#[cfg(test)] -mod tests { - use { - super::*, - serde_json::{Value, json}, - }; - - #[test] - fn serialize_query() { - assert_eq!( - serde_json::to_value(Query { - query: r#"foo { - }"#, - variables: Some(json_map! { - "foo" => "bar", - "baz" => 42, - "thing" => false, - }), - }) - .unwrap(), - json!({ - "query": "foo {\n }", - "variables": { - "foo": "bar", - "baz": 42, - "thing": false, - }, - }), - ); - } - - fn response_from_json(value: Value) -> Result - where - T: DeserializeOwned, - { - serde_json::from_value::>(value) - .unwrap() - .into_result() - } - - #[test] - fn deserialize_successful_response() { - assert!(response_from_json::(json!({ "data": true })).unwrap()); - } - - #[test] - fn deserialize_error_response() { - assert_eq!( - response_from_json::(json!({ - "data": null, - "errors": [{"message": "foo"}], - })) - .unwrap_err() - .to_string(), - "foo", - ); - assert_eq!( - response_from_json::(json!({ - "errors": [{"message": "bar"}], - })) - .unwrap_err() - .to_string(), - "bar", - ); - } - - #[test] - fn deserialize_multi_error_response() { - assert_eq!( - response_from_json::(json!({ - "data": null, - "errors": [ - {"message": "foo"}, - {"message": "bar"}, - ], - })) - .unwrap_err() - .to_string(), - "foo", - ); - } - - #[test] - fn deserialize_invalid_response() { - assert!( - response_from_json::(json!({ - "data": null, - "errors": null, - })) - .is_err() - ); - assert!( - response_from_json::(json!({ - "data": null, - "errors": [], - })) - .is_err() - ); - assert!( - response_from_json::(json!({ - "data": true, - "errors": [], - })) - .is_err() - ); - assert!( - response_from_json::(json!({ - "data": true, - "errors": [{"message":"bad"}], - })) - .is_err() - ); - } -} From d0f6001959287ddb6182beb847aed91f829336a6 Mon Sep 17 00:00:00 2001 From: Aryan Godara Date: Fri, 28 Aug 2026 01:40:17 +0530 Subject: [PATCH 7/7] add balancer-v2 ppool indexer e2e tests Signed-off-by: Aryan Godara --- crates/e2e/tests/e2e/pool_indexer.rs | 430 ++++++++++++++++++++++++++- 1 file changed, 423 insertions(+), 7 deletions(-) diff --git a/crates/e2e/tests/e2e/pool_indexer.rs b/crates/e2e/tests/e2e/pool_indexer.rs index 990b3cd098..9e10c66dd6 100644 --- a/crates/e2e/tests/e2e/pool_indexer.rs +++ b/crates/e2e/tests/e2e/pool_indexer.rs @@ -5,6 +5,7 @@ use { alloy::{ primitives::{ Address, + U256, aliases::{I24, U24, U160}, }, providers::Provider, @@ -16,6 +17,7 @@ use { number::units::EthUnit, pool_indexer::config::{ ApiConfig, + BalancerV2Config, Configuration, DatabaseConfig, FactoryConfig, @@ -127,6 +129,97 @@ sol! { } } +// Mock Balancer token. Bytecode compiled from the .sol below with solc 0.8.30, +// evm-version paris (no PUSH0, so it deploys at the node's pre-Shanghai +// genesis). +// +// contract MockToken { +// uint8 public decimals; +// constructor(uint8 d) { decimals = d; } +// } +sol! { + #[allow(missing_docs)] + #[sol(rpc, bytecode = "0x6080604052348015600f57600080fd5b506040516100ff3803806100ff833981016040819052602c916044565b6000805460ff191660ff92909216919091179055606c565b600060208284031215605557600080fd5b815160ff81168114606557600080fd5b9392505050565b60858061007a6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063313ce56714602d575b600080fd5b60005460399060ff1681565b60405160ff909116815260200160405180910390f3fea26469706673582212203e94d89ff9b8c4622833664d804ac0db01e8d6d8e741c93b5071047b13962bc364736f6c634300081e0033")] + contract MockToken { + constructor(uint8 d); + function decimals() external view returns (uint8); + } +} + +// Mock Balancer Vault. Compiled identically. +// +// contract MockBalancerVault { +// uint256 public nonce; +// mapping(bytes32 => address[]) tokens; +// mapping(bytes32 => uint256[]) balances; +// function registerPool() external returns (bytes32 id) { +// id = bytes32((uint256(uint160(msg.sender)) << 96) | nonce++); +// } +// function registerTokens( +// bytes32 id, address[] memory t, uint256[] memory b +// ) external { tokens[id] = t; balances[id] = b; } +// function getPoolTokens(bytes32 id) external view +// returns (address[] memory, uint256[] memory, uint256) +// { return (tokens[id], balances[id], 0); } +// } +sol! { + #[allow(missing_docs)] + #[sol(rpc, bytecode = "0x6080604052348015600f57600080fd5b506106158061001f6000396000f3fe608060405234801561001057600080fd5b506004361061004c5760003560e01c80637b09f30314610051578063affed0e014610066578063d7740ee114610082578063f94d46681461008a575b600080fd5b61006461005f3660046103da565b6100ac565b005b61006f60005481565b6040519081526020015b60405180910390f35b61006f6100f1565b61009d6100983660046104c7565b61010e565b604051610079939291906104e0565b600083815260016020908152604090912083516100cb928501906101f1565b50600083815260026020908152604090912082516100eb9284019061027b565b50505050565b60008054818061010083610580565b909155503360601b17919050565b6000818152600160209081526040808320600283528184208154835181860281018601909452808452606095869590948592909185919083018282801561018b57602002820191906000526020600020905b815473ffffffffffffffffffffffffffffffffffffffff168152600190910190602001808311610160575b50505050509250818054806020026020016040519081016040528092919081815260200182805480156101dd57602002820191906000526020600020905b8154815260200190600101908083116101c9575b505050505091509250925092509193909250565b82805482825590600052602060002090810192821561026b579160200282015b8281111561026b57825182547fffffffffffffffffffffffff00000000000000000000000000000000000000001673ffffffffffffffffffffffffffffffffffffffff909116178255602090920191600190910190610211565b506102779291506102b6565b5090565b82805482825590600052602060002090810192821561026b579160200282015b8281111561026b57825182559160200191906001019061029b565b5b8082111561027757600081556001016102b7565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe016810167ffffffffffffffff81118282101715610341576103416102cb565b604052919050565b600067ffffffffffffffff821115610363576103636102cb565b5060051b60200190565b600082601f83011261037e57600080fd5b813561039161038c82610349565b6102fa565b8082825260208201915060208360051b8601019250858311156103b357600080fd5b602085015b838110156103d05780358352602092830192016103b8565b5095945050505050565b6000806000606084860312156103ef57600080fd5b83359250602084013567ffffffffffffffff81111561040d57600080fd5b8401601f8101861361041e57600080fd5b803561042c61038c82610349565b8082825260208201915060208360051b85010192508883111561044e57600080fd5b6020840193505b8284101561049257833573ffffffffffffffffffffffffffffffffffffffff8116811461048157600080fd5b825260209384019390910190610455565b9450505050604084013567ffffffffffffffff8111156104b157600080fd5b6104bd8682870161036d565b9150509250925092565b6000602082840312156104d957600080fd5b5035919050565b6060808252845190820181905260009060208601906080840190835b8181101561053057835173ffffffffffffffffffffffffffffffffffffffff168352602093840193909201916001016104fc565b50508381036020808601919091528651808352918101925086019060005b8181101561056c57825184526020938401939092019160010161054e565b505050604092909201929092529392505050565b60007fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff82036105d8577f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b506001019056fea26469706673582212203197a76519149c61c64bfd3037184cfc3e2c3d6205ecff64443e2b0a2dec88de64736f6c634300081e0033")] + contract MockBalancerVault { + function getPoolTokens(bytes32 poolId) + external view returns (address[] memory, uint256[] memory, uint256); + } +} + +// Mock Balancer weighted-pool factory + the pool it deploys. Compiled +// identically. +// +// contract MockBalancerPool { +// bytes32 public poolId; +// uint256[] weights; +// constructor( +// MockBalancerVault v, address[] memory t, +// uint256[] memory w, uint256[] memory b +// ) { +// poolId = v.registerPool(); +// v.registerTokens(poolId, t, b); +// weights = w; +// } +// function getPoolId() external view returns (bytes32) { return poolId; } +// function getNormalizedWeights() +// external view returns (uint256[] memory) { return weights; } +// function getSwapFeePercentage() +// external pure returns (uint256) { return 1e15; } // 0.1% +// function getPausedState() +// external pure returns (bool, uint256, uint256) +// { return (false, 0, 0); } +// } +// +// contract MockBalancerPoolFactory { +// MockBalancerVault vault; +// event PoolCreated(address indexed pool); +// constructor(MockBalancerVault v) { vault = v; } +// function createPool( +// address[] memory t, uint256[] memory w, uint256[] memory b +// ) external returns (address pool) { +// pool = address(new MockBalancerPool(vault, t, w, b)); +// emit PoolCreated(pool); +// } +// } +sol! { + #[allow(missing_docs)] + #[sol(rpc, bytecode = "0x6080604052348015600f57600080fd5b50604051610ad1380380610ad1833981016040819052602c916050565b600080546001600160a01b0319166001600160a01b0392909216919091179055607e565b600060208284031215606157600080fd5b81516001600160a01b0381168114607757600080fd5b9392505050565b610a448061008d6000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c80637ac1eb8e1461003b578063fbfa77cf14610077575b600080fd5b61004e610049366004610257565b610097565b60405173ffffffffffffffffffffffffffffffffffffffff909116815260200160405180910390f35b60005461004e9073ffffffffffffffffffffffffffffffffffffffff1681565b6000805460405173ffffffffffffffffffffffffffffffffffffffff909116908590859085906100c69061013b565b6100d394939291906103a2565b604051809103906000f0801580156100ef573d6000803e3d6000fd5b5060405190915073ffffffffffffffffffffffffffffffffffffffff8216907f83a48fbcfc991335314e74d0496aab6a1987e992ddc85dddbcc4d6dd6ef2e9fc90600090a29392505050565b6105c98061044683390190565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe016810167ffffffffffffffff811182821017156101be576101be610148565b604052919050565b600067ffffffffffffffff8211156101e0576101e0610148565b5060051b60200190565b600082601f8301126101fb57600080fd5b813561020e610209826101c6565b610177565b8082825260208201915060208360051b86010192508583111561023057600080fd5b602085015b8381101561024d578035835260209283019201610235565b5095945050505050565b60008060006060848603121561026c57600080fd5b833567ffffffffffffffff81111561028357600080fd5b8401601f8101861361029457600080fd5b80356102a2610209826101c6565b8082825260208201915060208360051b8501019250888311156102c457600080fd5b6020840193505b8284101561030857833573ffffffffffffffffffffffffffffffffffffffff811681146102f757600080fd5b8252602093840193909101906102cb565b9550505050602084013567ffffffffffffffff81111561032757600080fd5b610333868287016101ea565b925050604084013567ffffffffffffffff81111561035057600080fd5b61035c868287016101ea565b9150509250925092565b600081518084526020840193506020830160005b8281101561039857815186526020958601959091019060010161037a565b5093949350505050565b60006080820173ffffffffffffffffffffffffffffffffffffffff871683526080602084015280865180835260a08501915060208801925060005b8181101561041157835173ffffffffffffffffffffffffffffffffffffffff168352602093840193909201916001016103dd565b505083810360408501526104258187610366565b915050828103606084015261043a8185610366565b97965050505050505056fe608060405234801561001057600080fd5b506040516105c93803806105c983398101604081905261002f91610264565b836001600160a01b031663d7740ee16040518163ffffffff1660e01b81526004016020604051808303816000875af115801561006f573d6000803e3d6000fd5b505050506040513d601f19601f82011682018060405250810190610093919061036f565b6000819055604051637b09f30360e01b81526001600160a01b03861691637b09f303916100c7919087908690600401610388565b600060405180830381600087803b1580156100e157600080fd5b505af11580156100f5573d6000803e3d6000fd5b5050835161010c9250600191506020850190610116565b505050505061041c565b828054828255906000526020600020908101928215610151579160200282015b82811115610151578251825591602001919060010190610136565b5061015d929150610161565b5090565b5b8082111561015d5760008155600101610162565b6001600160a01b038116811461018b57600080fd5b50565b634e487b7160e01b600052604160045260246000fd5b604051601f8201601f191681016001600160401b03811182821017156101cc576101cc61018e565b604052919050565b60006001600160401b038211156101ed576101ed61018e565b5060051b60200190565b600082601f83011261020857600080fd5b815161021b610216826101d4565b6101a4565b8082825260208201915060208360051b86010192508583111561023d57600080fd5b602085015b8381101561025a578051835260209283019201610242565b5095945050505050565b6000806000806080858703121561027a57600080fd5b845161028581610176565b60208601519094506001600160401b038111156102a157600080fd5b8501601f810187136102b257600080fd5b80516102c0610216826101d4565b8082825260208201915060208360051b8501019250898311156102e257600080fd5b6020840193505b8284101561030d5783516102fc81610176565b8252602093840193909101906102e9565b6040890151909650925050506001600160401b0381111561032d57600080fd5b610339878288016101f7565b606087015190935090506001600160401b0381111561035757600080fd5b610363878288016101f7565b91505092959194509250565b60006020828403121561038157600080fd5b5051919050565b6000606082018583526060602084015280855180835260808501915060208701925060005b818110156103d45783516001600160a01b03168352602093840193909201916001016103ad565b505083810360408501528451808252602091820192509085019060005b8181101561040f5782518452602093840193909201916001016103f1565b5091979650505050505050565b61019e8061042b6000396000f3fe608060405234801561001057600080fd5b50600436106100675760003560e01c80633e0dc34e116100505780633e0dc34e146100a257806355c67628146100ab578063f89f27ed146100b857600080fd5b80631c0de0511461006c57806338fff2d014610090575b600080fd5b60408051600080825260208201819052918101919091526060015b60405180910390f35b6000545b604051908152602001610087565b61009460005481565b66038d7ea4c68000610094565b6100c06100cd565b6040516100879190610125565b6060600180548060200260200160405190810160405280929190818152602001828054801561011b57602002820191906000526020600020905b815481526020019060010190808311610107575b5050505050905090565b602080825282518282018190526000918401906040840190835b8181101561015d57835183526020938401939092019160010161013f565b50909594505050505056fea26469706673582212208263ec4ec001151dc4ddf8de16bd3bd182b409db8ddd3052a793aded797c56dd64736f6c634300081e0033a2646970667358221220b108d4f0e3109c7d72ed7ff943b08bc35c01f9716f63de3414ca057482ccecb864736f6c634300081e0033")] + contract MockBalancerPoolFactory { + constructor(address vault); + event PoolCreated(address indexed pool); + function createPool( + address[] memory tokens, + uint256[] memory weights, + uint256[] memory balances, + ) external returns (address pool); + } +} + const POOL_INDEXER_PORT: u16 = 7778; const POOL_INDEXER_HOST: &str = "http://127.0.0.1:7778"; const POOL_INDEXER_METRICS_PORT: u16 = 7779; @@ -172,7 +265,7 @@ struct TickEntry {} async fn clear_pool_indexer_tables(db: &PgPool) { sqlx::query( "TRUNCATE uniswap_v3_ticks, uniswap_v3_pool_states, uniswap_v3_pools, \ - pool_indexer_checkpoints", + balancer_v2_pool_tokens, balancer_v2_pools, pool_indexer_checkpoints", ) .execute(db) .await @@ -232,11 +325,7 @@ fn pool_indexer_config( /// Spawns the pool-indexer task and waits for its `/health` endpoint to come /// up. -async fn spawn_pool_indexer( - factories: &[Address], - metrics_port: u16, -) -> tokio::task::JoinHandle<()> { - let config = pool_indexer_config(factories.iter().copied(), metrics_port); +async fn spawn_pool_indexer(config: Configuration) -> tokio::task::JoinHandle<()> { let handle = tokio::task::spawn(pool_indexer::run(config)); wait_for_condition(TIMEOUT, || async { reqwest::get(format!("{POOL_INDEXER_HOST}/health")) @@ -256,7 +345,8 @@ where F: FnOnce() -> Fut, Fut: Future, { - let handle = spawn_pool_indexer(factories, metrics_port).await; + let handle = + spawn_pool_indexer(pool_indexer_config(factories.iter().copied(), metrics_port)).await; let result = body().await; handle.abort(); let _ = handle.await; @@ -872,3 +962,329 @@ async fn min_envelope(_web3: Web3) { ) .await; } + +/// Pool-indexer config indexing the given balancer factories against `vault`. +fn balancer_pool_indexer_config( + vault: Address, + weighted: Vec
, + stable: Vec
, + metrics_port: u16, +) -> Configuration { + let factory = |address| FactoryConfig { + address, + deploy_block: 0, + }; + Configuration { + database: DatabaseConfig { + url: POOL_INDEXER_DB_URL.parse().unwrap(), + max_connections: NonZeroU32::new(5).unwrap(), + }, + network: NetworkConfig { + name: NetworkName::new("mainnet"), + chain_id: 1, + rpc_url: "http://127.0.0.1:8545".parse().unwrap(), + uniswap_v3: None, + balancer_v2: Some(BalancerV2Config { + vault, + chunk_size: 1000, + weighted: weighted.into_iter().map(factory).collect(), + weighted_v3plus: vec![], + stable: stable.into_iter().map(factory).collect(), + liquidity_bootstrapping: vec![], + composable_stable: vec![], + }), + poll_interval_secs: 1, + use_latest: true, + fetch_concurrency: 8, + prefetch_concurrency: 50, + }, + api: ApiConfig { + bind_address: SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, POOL_INDEXER_PORT)), + }, + metrics: MetricsConfig { + bind_address: SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, metrics_port)), + }, + } +} + +fn balancer_api(path: &str) -> String { + format!("{POOL_INDEXER_HOST}/api/v1/mainnet/balancer/v2/{path}") +} + +#[derive(Debug, Deserialize)] +struct BalancerPoolsListResponse { + block_number: u64, + pools: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct BalancerPoolResponse { + pool_type: String, + address: Address, + factory: Address, + swap_enabled: bool, + tokens: Vec, +} + +#[derive(Debug, Deserialize)] +struct BalancerTokenResponse { + address: Address, + decimals: u8, + #[serde(default)] + weight: Option, +} + +/// Creates a mock balancer pool and returns its address (from `PoolCreated`). +async fn create_balancer_pool( + factory: &MockBalancerPoolFactory::MockBalancerPoolFactoryInstance, + tokens: Vec
, + weights: Vec, + balances: Vec, +) -> Address { + let provider = factory.provider(); + factory + .createPool(tokens, weights, balances) + .send() + .await + .unwrap() + .watch() + .await + .unwrap(); + let block = provider.get_block_number().await.unwrap(); + let logs = provider + .get_logs( + &alloy::rpc::types::Filter::new() + .from_block(block) + .to_block(block) + .event_signature(MockBalancerPoolFactory::PoolCreated::SIGNATURE_HASH), + ) + .await + .unwrap(); + MockBalancerPoolFactory::PoolCreated::decode_log(&logs[0].inner) + .unwrap() + .data + .pool +} + +#[tokio::test] +#[ignore] +async fn local_node_pool_indexer_balancer_discovery() { + run_test(balancer_discovery).await; +} + +/// Asserts the indexer discovers a weighted + a stable pool and serves both +/// with the right type, token order, decimals, and weights (weighted only). +async fn balancer_discovery(web3: Web3) { + let db = PgPool::connect(POOL_INDEXER_DB_URL).await.unwrap(); + clear_pool_indexer_tables(&db).await; + let provider = web3.provider.clone().erased(); + + let vault = MockBalancerVault::deploy(provider.clone()).await.unwrap(); + let weighted_factory = MockBalancerPoolFactory::deploy(provider.clone(), *vault.address()) + .await + .unwrap(); + let stable_factory = MockBalancerPoolFactory::deploy(provider.clone(), *vault.address()) + .await + .unwrap(); + let token18 = MockToken::deploy(provider.clone(), 18u8).await.unwrap(); + let token6 = MockToken::deploy(provider.clone(), 6u8).await.unwrap(); + + // Weighted pool with 60/40 weights; stable pool carries no weights. + let weighted_tokens = vec![*token18.address(), *token6.address()]; + let weighted_pool = create_balancer_pool( + &weighted_factory, + weighted_tokens.clone(), + vec![ + U256::from(600_000_000_000_000_000u128), + U256::from(400_000_000_000_000_000u128), + ], + vec![U256::ZERO, U256::ZERO], + ) + .await; + let stable_tokens = vec![*token6.address(), *token18.address()]; + let stable_pool = create_balancer_pool( + &stable_factory, + stable_tokens.clone(), + vec![], + vec![U256::ZERO, U256::ZERO], + ) + .await; + + seed_checkpoint(&db, *weighted_factory.address(), 0).await; + seed_checkpoint(&db, *stable_factory.address(), 0).await; + let head = provider.get_block_number().await.unwrap(); + + let config = balancer_pool_indexer_config( + *vault.address(), + vec![*weighted_factory.address()], + vec![*stable_factory.address()], + POOL_INDEXER_METRICS_PORT, + ); + let handle = spawn_pool_indexer(config).await; + + wait_for_condition(TIMEOUT, || async { + let body: BalancerPoolsListResponse = reqwest::get(balancer_api("pools")) + .await + .ok()? + .json() + .await + .ok()?; + Some(body.block_number >= head && body.pools.len() >= 2) + }) + .await + .expect("indexer did not serve both balancer pools"); + + let body: BalancerPoolsListResponse = reqwest::get(balancer_api("pools")) + .await + .unwrap() + .json() + .await + .unwrap(); + let by_addr = |addr: Address| body.pools.iter().find(|p| p.address == addr).unwrap(); + + // Weighted: type + tokens (getPoolTokens order) + decimals + weights. + let w = by_addr(weighted_pool); + assert_eq!(w.pool_type, "Weighted"); + assert_eq!(w.factory, *weighted_factory.address()); + assert!(w.swap_enabled); + assert_eq!( + w.tokens.iter().map(|t| t.address).collect::>(), + weighted_tokens + ); + assert_eq!(w.tokens[0].decimals, 18); + assert_eq!(w.tokens[1].decimals, 6); + assert_eq!(w.tokens[0].weight.as_deref(), Some("0.6")); + assert_eq!(w.tokens[1].weight.as_deref(), Some("0.4")); + + // Stable: type + tokens (order) + decimals; no weights. + let s = by_addr(stable_pool); + assert_eq!(s.pool_type, "Stable"); + assert_eq!(s.factory, *stable_factory.address()); + assert_eq!( + s.tokens.iter().map(|t| t.address).collect::>(), + stable_tokens + ); + assert_eq!(s.tokens[0].decimals, 6); + assert_eq!(s.tokens[1].decimals, 18); + assert!(s.tokens.iter().all(|t| t.weight.is_none())); + + handle.abort(); + let _ = handle.await; +} + +#[tokio::test] +#[ignore] +async fn local_node_pool_indexer_balancer_driver_integration() { + run_test(balancer_driver_integration).await; +} + +/// Balancer analog of `driver_integration`: asserts (via the indexer's request +/// counter) that a driver with a `[[liquidity.balancer-v2]]` + `indexer-url` +/// source cold-reads the pool registry from the indexer at startup. Balancer +/// pool state is on-chain, so `/balancer/v2/pools` is the single cold-read +/// route (uni-v3 additionally serves ticks). +async fn balancer_driver_integration(web3: Web3) { + const POOLS_ROUTE: &str = "/api/v1/{network}/balancer/v2/pools"; + + let db = PgPool::connect(POOL_INDEXER_DB_URL).await.unwrap(); + clear_pool_indexer_tables(&db).await; + + let mut onchain = OnchainComponents::deploy(web3.clone()).await; + let [solver] = onchain.make_solvers(10u64.eth()).await; + let weth = *onchain.contracts().weth.address(); + + // WETH/token weighted pool via the mock factory the indexer scans and the + // driver's balancer config points at. + let provider = web3.provider.clone().erased(); + let vault = MockBalancerVault::deploy(provider.clone()).await.unwrap(); + let factory = MockBalancerPoolFactory::deploy(provider.clone(), *vault.address()) + .await + .unwrap(); + let token = MockToken::deploy(provider.clone(), 6u8).await.unwrap(); + create_balancer_pool( + &factory, + vec![weth, *token.address()], + vec![ + U256::from(500_000_000_000_000_000u128), + U256::from(500_000_000_000_000_000u128), + ], + vec![ + U256::from(100u128) * U256::from(10).pow(U256::from(18)), // 100 WETH + U256::from(300_000u128) * U256::from(10).pow(U256::from(6)), // 300k token + ], + ) + .await; + + let vault_addr = *vault.address(); + let factory_addr = *factory.address(); + seed_checkpoint(&db, factory_addr, 0).await; + let head = provider.get_block_number().await.unwrap(); + + let config = balancer_pool_indexer_config( + vault_addr, + vec![factory_addr], + vec![], + POOL_INDEXER_METRICS_PORT, + ); + let indexer = spawn_pool_indexer(config).await; + + wait_for_condition(TIMEOUT, || async { + let body: BalancerPoolsListResponse = reqwest::get(balancer_api("pools")) + .await + .ok()? + .json() + .await + .ok()?; + Some(body.block_number >= head && !body.pools.is_empty()) + }) + .await + .expect("indexer did not discover the balancer pool"); + + // Baseline AFTER warm-up so the bump below is driver-attributable. + let baseline_pools = api_requests_counter(POOL_INDEXER_METRICS_PORT, POOLS_ROUTE).await; + + let baseline_solver = colocation::start_baseline_solver( + "test_solver".into(), + solver.clone(), + weth, + vec![], + 1, + true, + ) + .await; + + let config_override = format!( + r#" +[[liquidity.balancer-v2]] +vault = "{vault_addr:?}" +weighted = ["{factory_addr:?}"] +indexer-url = "{POOL_INDEXER_HOST}" +"# + ); + let driver_handle = colocation::start_driver_with_config_override( + onchain.contracts(), + vec![baseline_solver], + colocation::LiquidityProvider::UniswapV2, + false, + Some(&config_override), + ); + + // The driver seeds its balancer registry from the indexer in the + // background, bumping the counter. + wait_for_condition(TIMEOUT, || async { + api_requests_counter(POOL_INDEXER_METRICS_PORT, POOLS_ROUTE).await > baseline_pools + }) + .await + .expect("driver did not cold-read balancer pools from the pool-indexer within timeout"); + + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM balancer_v2_pools") + .fetch_one(&db) + .await + .unwrap(); + assert!(count > 0, "expected balancer pools persisted to DB"); + + driver_handle.abort(); + indexer.abort(); + let _ = indexer.await; +}