feat(chain-sources): fetch headers at caller-requested heights (#2200)

crates/bitcoind_rpc/src/fetch_headers.rs

@@ -0,0 +1,128 @@
+//! On-demand header fetch at caller-requested heights.
+
+use bdk_core::collections::{BTreeMap, BTreeSet};
+use bdk_core::{bridge_heights, build_fetch_headers_update, CheckPoint};
+use bitcoin::block::Header;
+use bitcoincore_rpc::RpcApi;
+use core::ops::Deref;
+
+use alloc::vec::Vec;
+
+/// Error returned by [`fetch_headers_at_heights`].
+#[derive(Debug)]
+pub enum FetchHeadersError {
+    /// An RPC error occurred.
+    Rpc(bitcoincore_rpc::Error),
+    /// No common ancestor was found between `local_tip` and the node's best chain.
+    NoAgreement,
+}
+
+impl core::fmt::Display for FetchHeadersError {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        match self {
+            Self::Rpc(err) => write!(f, "bitcoind rpc error: {err}"),
+            Self::NoAgreement => write!(f, "no agreement point with bitcoind best chain"),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for FetchHeadersError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            Self::Rpc(err) => Some(err),
+            Self::NoAgreement => None,
+        }
+    }
+}
+
+impl From<bitcoincore_rpc::Error> for FetchHeadersError {
+    fn from(err: bitcoincore_rpc::Error) -> Self {
+        Self::Rpc(err)
+    }
+}
+
+/// Fetch a reorg-aware [`CheckPoint<Header>`] update covering `heights`.
+///
+/// Heights already present in `local_tip` are not fetched from the network. When every requested
+/// height is already known and no reorg is detected, returns `local_tip` unchanged (a no-op update
+/// that applies with an empty [`bdk_chain::local_chain::ChangeSet`]).
+///
+/// The returned checkpoint may include bridge blocks and reorg replacement headers beyond the
+/// requested set — only what is needed for [`bdk_chain::local_chain::LocalChain::apply_update`]
+/// to connect unambiguously.
+pub fn fetch_headers_at_heights<C>(
+    client: &C,
+    local_tip: CheckPoint<Header>,
+    heights: impl IntoIterator<Item = u32>,
+) -> Result<CheckPoint<Header>, FetchHeadersError>
+where
+    C: RpcApi,
+{
+    let requested: BTreeSet<u32> = heights.into_iter().collect();
+    let requested_vec: Vec<u32> = requested.iter().copied().collect();
+    let missing: Vec<u32> = requested
+        .iter()
+        .copied()
+        .filter(|h| local_tip.get(*h).is_none())
+        .collect();
+
+    let tip_height = client.get_block_count()? as u32;
+
+    let mut point_of_agreement = None;
+    let mut conflicts = Vec::new();
+
+    for local_cp in local_tip.iter() {
+        let height = local_cp.height();
+        if height > tip_height {
+            continue;
+        }
+        let remote_hash = client.get_block_hash(height as u64)?;
+        if remote_hash == local_cp.hash() {
+            point_of_agreement = Some(local_cp);
+            break;
+        }
+        let header = client.get_block_header(&remote_hash)?;
+        conflicts.push((height, header));
+    }
+
+    let agreement = match point_of_agreement {
+        Some(cp) => cp,
+        None => return Err(FetchHeadersError::NoAgreement),
+    };
+
+    let reorg_detected = !conflicts.is_empty();
+    if missing.is_empty() && !reorg_detected {
+        return Ok(local_tip);
+    }
+
+    let mut fetched = BTreeMap::new();
+    for &height in &missing {
+        if height > tip_height {
+            return Err(FetchHeadersError::Rpc(
+                client.get_block_hash(height as u64).unwrap_err(),
+            ));
+        }
+        let hash = client.get_block_hash(height as u64)?;
+        let header = client.get_block_header(&hash)?;
+        fetched.insert(height, header);
+    }
+
+    let bridges = bridge_heights(&local_tip, agreement.height(), &requested_vec);
+    Ok(build_fetch_headers_update(
+        agreement, conflicts, fetched, &local_tip, &bridges,
+    ))
+}
+
+/// Fetch headers at caller-requested heights using a dereferencing client wrapper.
+pub fn fetch_headers_at_heights_with<C>(
+    client: C,
+    local_tip: CheckPoint<Header>,
+    heights: impl IntoIterator<Item = u32>,
+) -> Result<CheckPoint<Header>, FetchHeadersError>
+where
+    C: Deref,
+    C::Target: RpcApi,
+{
+    fetch_headers_at_heights(&*client, local_tip, heights)
+}

crates/bitcoind_rpc/src/lib.rs

@@ -22,8 +22,12 @@ use bitcoincore_rpc::{bitcoincore_rpc_json, RpcApi};
 use core::ops::Deref;
 
 pub mod bip158;
+mod fetch_headers;
 
 pub use bitcoincore_rpc;
+pub use fetch_headers::{
+    fetch_headers_at_heights, fetch_headers_at_heights_with, FetchHeadersError,
+};
 
 /// The [`Emitter`] is used to emit data sourced from [`bitcoincore_rpc::Client`].
 ///

crates/bitcoind_rpc/tests/test_fetch_headers.rs

@@ -0,0 +1,150 @@
+use std::collections::BTreeMap;
+
+use bdk_bitcoind_rpc::{fetch_headers_at_heights, FetchHeadersError};
+use bdk_chain::local_chain::{ChangeSet, LocalChain};
+use bdk_testenv::{anyhow, TestEnv};
+use bitcoin::block::Header;
+
+use crate::common::ClientExt;
+
+mod common;
+
+fn header_at(env: &TestEnv, height: u32) -> anyhow::Result<Header> {
+    let hash = env.get_block_hash(height as u64)?;
+    Ok(env.rpc_client().get_block_header(&hash)?.into_model()?.0)
+}
+
+fn local_chain_from_headers(blocks: &[(u32, Header)]) -> LocalChain<Header> {
+    let map: BTreeMap<u32, Header> = blocks.iter().copied().collect();
+    LocalChain::from_blocks(map).expect("valid sparse chain")
+}
+
+#[test]
+fn fetch_headers_at_heights_exact_heights() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    env.mine_blocks(30, None)?;
+
+    let genesis = header_at(&env, 0)?;
+    let h21 = header_at(&env, 21)?;
+    let mut chain = local_chain_from_headers(&[(0, genesis), (21, h21)]);
+
+    let update =
+        fetch_headers_at_heights(&ClientExt::get_rpc_client(&env)?, chain.tip(), [22, 25, 28])?;
+
+    for h in [22, 25, 28] {
+        assert!(update.get(h).is_some(), "update must contain height {h}");
+    }
+
+    chain.apply_update(update)?;
+    for h in [22, 25, 28] {
+        assert_eq!(chain.get(h).map(|cp| cp.height()), Some(h));
+    }
+
+    Ok(())
+}
+
+#[test]
+fn fetch_headers_at_heights_skip_already_known() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    env.mine_blocks(10, None)?;
+
+    let blocks: Vec<(u32, Header)> = (0..=5)
+        .map(|h| Ok((h, header_at(&env, h)?)))
+        .collect::<anyhow::Result<_>>()?;
+    let mut chain = local_chain_from_headers(&blocks);
+
+    let update =
+        fetch_headers_at_heights(&ClientExt::get_rpc_client(&env)?, chain.tip(), [3, 4, 5, 6])?;
+
+    assert!(update.get(6).is_some());
+
+    let changeset = chain.apply_update(update)?;
+    assert_eq!(changeset.blocks.get(&3), None);
+    assert_eq!(changeset.blocks.get(&4), None);
+    assert_eq!(changeset.blocks.get(&5), None);
+    assert!(changeset.blocks.contains_key(&6));
+
+    Ok(())
+}
+
+#[test]
+fn fetch_headers_at_heights_no_op_empty_changeset() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    env.mine_blocks(5, None)?;
+
+    let blocks: Vec<(u32, Header)> = (0..=3)
+        .map(|h| Ok((h, header_at(&env, h)?)))
+        .collect::<anyhow::Result<_>>()?;
+    let chain = local_chain_from_headers(&blocks);
+
+    let update =
+        fetch_headers_at_heights(&ClientExt::get_rpc_client(&env)?, chain.tip(), [1, 2, 3])?;
+
+    assert_eq!(update, chain.tip());
+
+    let mut chain = chain;
+    let changeset = chain.apply_update(update)?;
+    assert_eq!(changeset, ChangeSet::default());
+
+    Ok(())
+}
+
+#[test]
+fn fetch_headers_at_heights_mtp_recovery() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    env.mine_blocks(30, None)?;
+
+    let genesis = header_at(&env, 0)?;
+    let h21 = header_at(&env, 21)?;
+    let mut chain = local_chain_from_headers(&[(0, genesis), (21, h21)]);
+
+    assert_eq!(chain.get(21).unwrap().median_time_past(), None);
+
+    let missing: Vec<u32> = (11..=21).collect();
+    let update = fetch_headers_at_heights(&ClientExt::get_rpc_client(&env)?, chain.tip(), missing)?;
+
+    chain.apply_update(update)?;
+    assert!(chain.get(21).unwrap().median_time_past().is_some());
+
+    Ok(())
+}
+
+#[test]
+fn fetch_headers_at_heights_sparse_gap_bridge() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    env.mine_blocks(10, None)?;
+
+    let blocks = [
+        (0, header_at(&env, 0)?),
+        (1, header_at(&env, 1)?),
+        (5, header_at(&env, 5)?),
+    ];
+    let mut chain = local_chain_from_headers(&blocks);
+
+    let update = fetch_headers_at_heights(&ClientExt::get_rpc_client(&env)?, chain.tip(), [4])?;
+
+    assert!(update.get(4).is_some());
+    assert!(update.get(5).is_some());
+
+    chain.apply_update(update)?;
+    assert_eq!(chain.get(4).map(|cp| cp.height()), Some(4));
+
+    Ok(())
+}
+
+#[test]
+fn fetch_headers_at_heights_height_above_tip_errors() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    env.mine_blocks(5, None)?;
+
+    let genesis = header_at(&env, 0)?;
+    let chain = local_chain_from_headers(&[(0, genesis)]);
+    let tip = env.rpc_client().get_block_count()?.into_model().0 as u32;
+
+    let err = fetch_headers_at_heights(&ClientExt::get_rpc_client(&env)?, chain.tip(), [tip + 100])
+        .unwrap_err();
+
+    assert!(matches!(err, FetchHeadersError::Rpc(_)));
+
+    Ok(())
+}

crates/core/src/fetch_headers_update.rs

@@ -0,0 +1,101 @@
+//! Helpers for building sparse [`CheckPoint<Header>`] updates from on-demand height fetches.
+
+use alloc::collections::{BTreeMap, BTreeSet};
+use alloc::vec::Vec;
+
+use bitcoin::block::Header;
+
+use crate::CheckPoint;
+
+/// For each requested height, the nearest local checkpoint strictly above it (if any).
+///
+/// Bridge blocks connect sparse gaps so [`bdk_chain::local_chain::LocalChain::apply_update`]
+/// can merge the update unambiguously.
+pub fn bridge_heights(
+    local_tip: &CheckPoint<Header>,
+    agreement_height: u32,
+    requested_heights: &[u32],
+) -> BTreeSet<u32> {
+    let mut bridges = BTreeSet::new();
+    for &h in requested_heights {
+        for local_cp in local_tip.iter() {
+            let lh = local_cp.height();
+            if lh > h && lh > agreement_height {
+                bridges.insert(lh);
+                break;
+            }
+        }
+    }
+    bridges
+}
+
+/// Assemble a sparse header checkpoint update after the backend has resolved agreement and fetches.
+///
+/// `conflicts` must be in descending height order (as collected from an agreement walk from tip).
+pub fn build_fetch_headers_update(
+    agreement: CheckPoint<Header>,
+    conflicts: Vec<(u32, Header)>,
+    fetched: BTreeMap<u32, Header>,
+    local_tip: &CheckPoint<Header>,
+    bridge_heights: &BTreeSet<u32>,
+) -> CheckPoint<Header> {
+    let mut tip = agreement;
+
+    if !conflicts.is_empty() {
+        tip = tip
+            .extend(conflicts.into_iter().rev())
+            .expect("conflict headers must be in ascending height order");
+    }
+
+    for (height, header) in fetched {
+        if tip.get(height).is_none() {
+            tip = tip.insert(height, header);
+        }
+    }
+
+    for &height in bridge_heights {
+        if tip.get(height).is_none() {
+            if let Some(cp) = local_tip.get(height) {
+                tip = tip.insert(height, cp.data());
+            }
+        }
+    }
+
+    tip
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bitcoin::block::Header;
+    use bitcoin::hashes::Hash;
+    use bitcoin::BlockHash;
+    use bitcoin::CompactTarget;
+
+    fn dummy_header(prev: BlockHash, n: u32) -> Header {
+        Header {
+            version: bitcoin::block::Version::from_consensus(1),
+            prev_blockhash: prev,
+            merkle_root: bitcoin::hashes::Hash::hash(&n.to_le_bytes()),
+            time: 1_000 + n,
+            bits: CompactTarget::from_consensus(0),
+            nonce: n,
+        }
+    }
+
+    fn sparse_local_chain() -> CheckPoint<Header> {
+        let h0 = dummy_header(BlockHash::all_zeros(), 0);
+        let h1 = dummy_header(h0.block_hash(), 1);
+        let h5 = dummy_header(h1.block_hash(), 5);
+        CheckPoint::new(0, h0)
+            .insert(1, h1)
+            .insert(5, h5)
+    }
+
+    #[test]
+    fn bridge_heights_picks_nearest_above() {
+        let local = sparse_local_chain();
+        let bridges = bridge_heights(&local, 1, &[4]);
+        assert_eq!(bridges, BTreeSet::from([5]));
+    }
+}