From 13eb39cf959f3ebbdcaa58572ed6659fd15fb877 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Wed, 9 Sep 2026 16:46:01 +0300 Subject: [PATCH 1/2] feat(scenarios): add reusable raw-layout overrides --- crates/core/src/scenarios/README.md | 23 ++ crates/core/src/scenarios/registry.rs | 85 ++++- crates/core/src/surfnet/svm.rs | 67 +++- crates/core/src/tests/kamino/mod.rs | 32 +- crates/core/src/tests/pump/mod.rs | 21 +- crates/types/src/scenarios.rs | 480 +++++++++++++++++++++++++- 6 files changed, 665 insertions(+), 43 deletions(-) diff --git a/crates/core/src/scenarios/README.md b/crates/core/src/scenarios/README.md index 70e080af2..6a53395d5 100644 --- a/crates/core/src/scenarios/README.md +++ b/crates/core/src/scenarios/README.md @@ -56,6 +56,29 @@ cargo test -p surfpool-core --features integration-tests kamino Set `SURFPOOL_TEST_RPC_URL` to use a private endpoint instead of the public one. The default test run needs no network. +### Programs with no IDL + +Programs that publish no usable IDL can describe their account bytes directly in an override +collection: + +```yaml +raw_layout: + account_size: 128 + magic: { offset: 0, bytes: [69, 88, 65, 77, 80, 76, 69] } + +templates: + - id: example-market-price + properties: + - path: price + offset: 32 + encoding: u128 +``` + +Raw-layout overrides use the same scenario API as IDL-backed templates. Values are encoded as +little-endian integers and can be written once or repeatedly at a fixed stride. Account-size and +magic guards ensure a template refuses the wrong account before changing any bytes. Large integers +that exceed `u64` should be supplied as decimal strings so JSON parsing cannot lose precision. + ### Override Templates Directly using the `surfnet_registerScenario` endpoint requires building out a map of account keys that are specific to the schema of the account that is being written to. This is a cumbersome process in most cases. diff --git a/crates/core/src/scenarios/registry.rs b/crates/core/src/scenarios/registry.rs index 217ac8039..11f419e19 100644 --- a/crates/core/src/scenarios/registry.rs +++ b/crates/core/src/scenarios/registry.rs @@ -189,7 +189,21 @@ impl TemplateRegistry { Ok(idl) => idl, Err(e) => panic!("unable to load {} idl: {}", protocol_name, e), }; + self.load_collection(Some(idl), overrides_content, protocol_name); + } + + /// For programs that publish no IDL. Their templates must carry a `raw_layout` and spell out + /// every property description, since there is no schema to fall back on. + pub fn load_raw_layout_overrides(&mut self, overrides_content: &str, protocol_name: &str) { + self.load_collection(None, overrides_content, protocol_name); + } + fn load_collection( + &mut self, + idl: Option, + overrides_content: &str, + protocol_name: &str, + ) { let collection = match serde_yaml::from_str::(overrides_content) { Ok(c) => c, @@ -557,6 +571,49 @@ mod tests { assert!(registry.contains("pump-amm-global-config")); } + #[test] + fn raw_layout_collection_loads_without_an_idl() { + const OVERRIDES: &str = r#" +protocol: Example +version: v1 +account_type: State +raw_layout: + account_size: 16 + magic: + offset: 0 + bytes: [69, 88] +templates: + - id: example-raw-value + name: Override Value + description: Override one integer in an example binary account + address: + type: pubkey + value: "11111111111111111111111111111111" + properties: + - path: value + offset: 8 + encoding: u64 + label: Value + description: Example unsigned integer +"#; + + let mut registry = TemplateRegistry::default(); + registry.load_raw_layout_overrides(OVERRIDES, "example"); + + let template = registry.get("example-raw-value").expect("raw template"); + assert!(template.idl.is_none()); + let layout = template.raw_layout.as_ref().expect("raw layout"); + let output = layout + .materialize( + &[69, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + &template.properties, + &HashMap::from([("value".to_string(), serde_json::json!(42))]), + 0, + ) + .expect("materialize raw template"); + assert_eq!(u64::from_le_bytes(output[8..16].try_into().unwrap()), 42); + } + #[test] fn test_jupiter_template_loads_correctly() { let registry = TemplateRegistry::new(); @@ -717,7 +774,7 @@ mod tests { let registry = TemplateRegistry::new(); let jupiter_template = registry.get("jupiter-token-ledger-override").unwrap(); let has_token_ledger = jupiter_template - .idl + .idl() .accounts .iter() .any(|acc| acc.name == "TokenLedger"); @@ -1157,18 +1214,24 @@ mod tests { let registry = TemplateRegistry::new(); let mut errors = Vec::new(); + let mut checked = 0usize; for template in registry.all() { + // Templates for programs that publish no IDL declare their own byte offsets, so there + // is no schema for their paths to resolve against. Raw encoding and guarded writes are + // covered by the unit tests on `RawLayout`. + let Some(idl) = template.idl.as_ref() else { + continue; + }; for property in &template.properties { // constant_ref properties are UI dropdowns (e.g. token pickers), not // account fields, so they are not expected to resolve against the IDL. if property.is_constant_ref() { continue; } - if let Err(e) = surfpool_types::resolve_idl_type( - &template.idl, - &template.account_type, - &property.path, - ) { + checked += 1; + if let Err(e) = + surfpool_types::resolve_idl_type(idl, &template.account_type, &property.path) + { errors.push(format!("[{}] {}: {}", template.id, property.path, e)); } } @@ -1180,6 +1243,12 @@ mod tests { errors.len(), errors.join("\n ") ); + // Without this the skip above could silently swallow every template and the test would pass + // having resolved nothing. + assert!( + checked > 0, + "no property was resolved against an IDL, so this proved nothing" + ); } #[test] @@ -1339,7 +1408,7 @@ mod tests { ("ref_price.0", IdlType::U16), ] { let resolved = - surfpool_types::resolve_idl_type(&template.idl, &template.account_type, path) + surfpool_types::resolve_idl_type(template.idl(), &template.account_type, path) .unwrap_or_else(|e| panic!("{path} should resolve: {e}")); assert_eq!( *resolved, expected, @@ -1352,7 +1421,7 @@ mod tests { .get("kamino-obligation-positions") .expect("kamino-obligation-positions should exist"); let resolved = surfpool_types::resolve_idl_type( - &obligation.idl, + obligation.idl(), &obligation.account_type, "deposits.0.deposit_reserve", ) diff --git a/crates/core/src/surfnet/svm.rs b/crates/core/src/surfnet/svm.rs index baff46c76..831432580 100644 --- a/crates/core/src/surfnet/svm.rs +++ b/crates/core/src/surfnet/svm.rs @@ -285,6 +285,13 @@ fn json_integer_digits(json: &serde_json::Value, target: &str) -> SurfpoolResult } } +/// The bundled template registry, parsed once and reused. +fn template_registry() -> &'static crate::scenarios::TemplateRegistry { + static REGISTRY: std::sync::OnceLock = + std::sync::OnceLock::new(); + REGISTRY.get_or_init(crate::scenarios::TemplateRegistry::new) +} + /// Converts JSON into a txtx [`Value`] using the expected IDL type fn json_to_txtx_value_for_idl_type( json: &serde_json::Value, @@ -841,7 +848,11 @@ impl SurfnetSvm { fn register_builtin_template_idls(&mut self) { let registry = TemplateRegistry::new(); for (_, template) in registry.templates.into_iter() { - let _ = self.register_idl(template.idl, None); + // Templates for programs with no IDL have nothing to register; they write through + // `raw_layout` instead. + if let Some(idl) = template.idl { + let _ = self.register_idl(idl, None); + } } } @@ -3065,6 +3076,50 @@ impl SurfnetSvm { continue; }; + // Programs with no usable IDL carry a byte layout instead, and this MUST come + // before the IDL lookup below: those programs have no registered IDL at all, so the + // lookup would `continue` and silently drop the override. + let raw_template = template_registry() + .get(&override_instance.template_id) + .filter(|t| t.raw_layout.is_some()) + .cloned(); + if let Some(template) = raw_template { + let raw_layout = template.raw_layout.expect("filtered above"); + let properties = template.properties; + match raw_layout.materialize( + account.data(), + &properties, + &account_values, + target_slot, + ) { + Ok(new_data) => { + let modified = Account { + lamports: account.lamports(), + data: new_data, + owner: *account.owner(), + executable: account.executable(), + rent_epoch: account.rent_epoch(), + }; + if let Err(e) = self.inner.set_account(account_pubkey, modified) { + warn!("Failed to set raw-layout account {}: {}", account_pubkey, e); + } else { + debug!( + "Raw-layout override {} applied {} field(s) to {}", + override_instance.id, + account_values.len(), + account_pubkey + ); + settled_this_slot.insert(account_pubkey); + } + } + Err(e) => warn!( + "Raw-layout override {} failed on {}: {}", + override_instance.id, account_pubkey, e + ), + } + continue; + } + // Mints fail the token unpack and keep flowing through the IDL path. if is_supported_token_program(account.owner()) { if let Ok(token_account) = TokenAccount::unpack(account.data()) { @@ -5809,10 +5864,18 @@ mod tests { assert!(!epoch_schedule.warmup); let registry = TemplateRegistry::new(); + let mut checked = 0usize; for (_, template) in registry.templates { - let program_id = template.idl.address.clone(); + // Templates for programs that publish no IDL have nothing to register. + let Some(idl) = template.idl else { continue }; + let program_id = idl.address.clone(); assert!(svm.registered_idls.get(&program_id).unwrap().is_some()); + checked += 1; } + assert!( + checked > 0, + "no template carried an IDL, so this proved nothing about registration" + ); assert!(svm.skip_blockhash_check); } diff --git a/crates/core/src/tests/kamino/mod.rs b/crates/core/src/tests/kamino/mod.rs index 9bdc0c1a5..aef2ba25e 100644 --- a/crates/core/src/tests/kamino/mod.rs +++ b/crates/core/src/tests/kamino/mod.rs @@ -107,7 +107,7 @@ async fn real_mainnet_accounts_round_trip_unchanged() { .unwrap_or_else(|| panic!("template {template_id} should exist")); let account_def = template - .idl + .idl() .accounts .iter() .find(|a| a.name == *account_name) @@ -119,7 +119,7 @@ async fn real_mainnet_accounts_round_trip_unchanged() { ); let forged = surfnet_svm - .get_forged_account_data(&pubkey, data, &template.idl, &HashMap::new()) + .get_forged_account_data(&pubkey, data, template.idl(), &HashMap::new()) .unwrap_or_else(|e| { panic!( "live mainnet {account_name} failed to decode/re-encode with the bundled \ @@ -167,7 +167,7 @@ async fn override_on_real_account_touches_only_target_bytes() { .get_forged_account_data( &pubkey, reserve_data, - &reserve.idl, + reserve.idl(), &HashMap::from([( "config.liquidation_threshold_pct".to_string(), serde_json::json!(50u8), @@ -201,7 +201,7 @@ async fn override_on_real_account_touches_only_target_bytes() { .get_forged_account_data( &pubkey, scope_data, - &scope.idl, + scope.idl(), &HashMap::from([( format!("prices.{IDX}.price.value"), serde_json::json!(new_value), @@ -359,7 +359,7 @@ async fn every_template_round_trips_over_a_live_account() { .filter(|t| t.account_type == *account_type) { let identity = surfnet_svm - .get_forged_account_data(&pubkey, data, &template.idl, &HashMap::new()) + .get_forged_account_data(&pubkey, data, template.idl(), &HashMap::new()) .unwrap_or_else(|e| { panic!( "identity round-trip failed for {} ({address}): {e}", @@ -383,7 +383,7 @@ async fn every_template_round_trips_over_a_live_account() { let mut overrides: HashMap = HashMap::new(); for property in &template.properties { let ty = surfpool_types::resolve_idl_type( - &template.idl, + template.idl(), &template.account_type, &property.path, ) @@ -397,7 +397,7 @@ async fn every_template_round_trips_over_a_live_account() { } let forged = surfnet_svm - .get_forged_account_data(&pubkey, data, &template.idl, &overrides) + .get_forged_account_data(&pubkey, data, template.idl(), &overrides) .unwrap_or_else(|e| { panic!( "forge failed for {} with {} scalar override(s): {e}", @@ -460,7 +460,7 @@ async fn obligation_array_index_and_pubkey_overrides() { ]); let forged = surfnet_svm - .get_forged_account_data(&Pubkey::new_unique(), &data, &template.idl, &overrides) + .get_forged_account_data(&Pubkey::new_unique(), &data, template.idl(), &overrides) .expect("array-index and pubkey overrides should apply"); assert_eq!(forged.len(), data.len(), "account size must be preserved"); @@ -524,7 +524,7 @@ async fn scope_price_override_writes_expected_bytes() { ]); let forged = surfnet_svm - .get_forged_account_data(&Pubkey::new_unique(), &data, &template.idl, &overrides) + .get_forged_account_data(&Pubkey::new_unique(), &data, template.idl(), &overrides) .expect("scope price override should apply"); assert_eq!(forged.len(), data.len(), "account size must be preserved"); @@ -574,7 +574,7 @@ async fn farms_reward_override_writes_both_halves() { ), ]); let forged_farm = surfnet_svm - .get_forged_account_data(&pubkey, farm_data, &farm.idl, &farm_overrides) + .get_forged_account_data(&pubkey, farm_data, farm.idl(), &farm_overrides) .expect("farm accumulator override should apply"); assert_eq!(forged_farm.len(), farm_data.len()); assert_ne!(&forged_farm, farm_data); @@ -601,7 +601,7 @@ async fn farms_reward_override_writes_both_halves() { ), ]); let forged_user = surfnet_svm - .get_forged_account_data(&pubkey, user_data, &user.idl, &user_overrides) + .get_forged_account_data(&pubkey, user_data, user.idl(), &user_overrides) .expect("user reward override should apply"); assert_eq!(forged_user.len(), user_data.len()); @@ -644,7 +644,7 @@ async fn liquidation_setup_writes_durable_inputs() { (format!("prices.{IDX}.price.exp"), serde_json::json!(8u64)), ]); let forged_scope = surfnet_svm - .get_forged_account_data(&pubkey, scope_data, &scope.idl, &scope_overrides) + .get_forged_account_data(&pubkey, scope_data, scope.idl(), &scope_overrides) .expect("scope crash should apply"); let off = SCOPE_PRICES_BASE + IDX * DATED_PRICE_SIZE; @@ -675,7 +675,7 @@ async fn liquidation_setup_writes_durable_inputs() { ), ]); let forged_reserve = surfnet_svm - .get_forged_account_data(&pubkey, reserve_data, &reserve.idl, &reserve_overrides) + .get_forged_account_data(&pubkey, reserve_data, reserve.idl(), &reserve_overrides) .expect("reserve config override should apply"); assert_eq!( @@ -709,7 +709,7 @@ async fn withdraw_ticket_and_queue_cursor() { .get("kamino-withdraw-ticket") .expect("withdraw ticket template"); let ticket_disc = &ticket - .idl + .idl() .accounts .iter() .find(|a| a.name == "WithdrawTicket") @@ -727,7 +727,7 @@ async fn withdraw_ticket_and_queue_cursor() { ("invalid".to_string(), serde_json::json!(0u8)), ]); let forged_ticket = surfnet_svm - .get_forged_account_data(&pubkey, &ticket_data, &ticket.idl, &ticket_overrides) + .get_forged_account_data(&pubkey, &ticket_data, ticket.idl(), &ticket_overrides) .expect("withdraw ticket override should apply"); assert_eq!( u64::from_le_bytes(forged_ticket[8..16].try_into().unwrap()), @@ -758,7 +758,7 @@ async fn withdraw_ticket_and_queue_cursor() { ), ]); let forged_reserve = surfnet_svm - .get_forged_account_data(&pubkey, &reserve_data, &limits.idl, &queue_overrides) + .get_forged_account_data(&pubkey, &reserve_data, limits.idl(), &queue_overrides) .expect("withdraw queue override should apply"); assert_eq!(forged_reserve.len(), reserve_data.len()); diff --git a/crates/core/src/tests/pump/mod.rs b/crates/core/src/tests/pump/mod.rs index 960ce8e2f..650662766 100644 --- a/crates/core/src/tests/pump/mod.rs +++ b/crates/core/src/tests/pump/mod.rs @@ -157,9 +157,12 @@ async fn real_mainnet_accounts_round_trip_unchanged() { let template = registry .get(template_id) .unwrap_or_else(|| panic!("template {template_id} should exist")); - - let account_def = template + let idl = template .idl + .as_ref() + .unwrap_or_else(|| panic!("Pump template {template_id} must carry an IDL")); + + let account_def = idl .accounts .iter() .find(|a| a.name == *account_name) @@ -171,7 +174,7 @@ async fn real_mainnet_accounts_round_trip_unchanged() { ); let forged = surfnet_svm - .get_forged_account_data(&pubkey, &account.data, &template.idl, &HashMap::new()) + .get_forged_account_data(&pubkey, &account.data, idl, &HashMap::new()) .unwrap_or_else(|e| { panic!( "live mainnet {account_name} {address} failed to decode/re-encode with the \ @@ -226,8 +229,12 @@ async fn override_on_real_account_touches_only_target_bytes() { ), ("complete".to_string(), serde_json::json!(true)), ]); + let curve_idl = curve + .idl + .as_ref() + .expect("Pump curve template must carry an IDL"); let forged = surfnet_svm - .get_forged_account_data(&pubkey, curve_data, &curve.idl, &overrides) + .get_forged_account_data(&pubkey, curve_data, curve_idl, &overrides) .expect("curve override on the live bonding curve"); assert_eq!( forged.len(), @@ -265,8 +272,12 @@ async fn override_on_real_account_touches_only_target_bytes() { serde_json::json!(5_000_000_000i64), ), ]); + let pool_idl = pool + .idl + .as_ref() + .expect("Pump AMM template must carry an IDL"); let forged = surfnet_svm - .get_forged_account_data(&pubkey, pool_data, &pool.idl, &overrides) + .get_forged_account_data(&pubkey, pool_data, pool_idl, &overrides) .expect("pool override on the live canonical pool"); assert_eq!( forged.len(), diff --git a/crates/types/src/scenarios.rs b/crates/types/src/scenarios.rs index a04ca80a1..0e25a7a4a 100644 --- a/crates/types/src/scenarios.rs +++ b/crates/types/src/scenarios.rs @@ -277,6 +277,12 @@ pub struct Property { /// For constant_ref type: the name of the constant definition to use #[serde(default, skip_serializing_if = "Option::is_none")] pub constant: Option, + /// Raw-layout only: byte offset of this field within the account. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offset: Option, + /// Raw-layout only: how this field's bytes are produced. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encoding: Option, } impl Property { @@ -288,6 +294,8 @@ impl Property { label: None, description: None, constant: None, + offset: None, + encoding: None, } } @@ -299,6 +307,8 @@ impl Property { label: None, description: None, constant: Some(constant.into()), + offset: None, + encoding: None, } } @@ -383,8 +393,11 @@ pub struct OverrideTemplate { pub description: String, /// Protocol this template is for (e.g., "Pyth", "Switchboard") pub protocol: String, - /// IDL for the account structure - defines all available fields and types - pub idl: Idl, + /// IDL for the account structure - defines all available fields and types. + /// + /// `None` for programs that publish no IDL and are written through `raw_layout` instead. Those + /// templates cannot use the IDL write path at all, so there is nothing to reconstruct here. + pub idl: Option, /// How to determine the account address pub address: AccountAddress, /// Account type name from the IDL (e.g., "PriceAccount") @@ -401,9 +414,27 @@ pub struct OverrideTemplate { /// This helps LLMs understand how to correctly use the template #[serde(default, skip_serializing_if = "Option::is_none")] pub llm_context: Option, + /// Set for programs with no usable IDL. When present the override engine writes bytes at + /// each property's offset instead of decoding and re-encoding through the IDL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub raw_layout: Option, } impl OverrideTemplate { + /// The IDL this template was built from. + /// + /// Panics for templates that have none - those belong to programs that publish no IDL and are + /// written through `raw_layout`. Callers that may legitimately see either must match on the + /// field instead of calling this. + pub fn idl(&self) -> &Idl { + self.idl.as_ref().unwrap_or_else(|| { + panic!( + "template {} has no IDL; it is written through raw_layout", + self.id + ) + }) + } + pub fn new( id: String, name: String, @@ -419,13 +450,14 @@ impl OverrideTemplate { name, description, protocol, - idl, + idl: Some(idl), address, account_type, properties, constants: HashMap::new(), tags: Vec::new(), llm_context: None, + raw_layout: None, } } @@ -642,7 +674,8 @@ pub struct YamlOverrideTemplateFile { pub properties: Vec, #[serde(default)] pub constants: HashMap, - pub idl_file_path: String, + #[serde(default)] + pub idl_file_path: Option, pub address: YamlAccountAddress, #[serde(default)] pub tags: Vec, @@ -659,7 +692,7 @@ impl YamlOverrideTemplateFile { name: self.name, description: self.description, protocol: self.protocol, - idl, + idl: Some(idl), address: self.address.into(), account_type: self.account_type, properties: self.properties.into_iter().map(Into::into).collect(), @@ -670,6 +703,7 @@ impl YamlOverrideTemplateFile { .collect(), tags: self.tags, llm_context: self.llm_context, + raw_layout: None, } } } @@ -850,6 +884,12 @@ pub enum YamlProperty { /// For constant_ref type: the name of the constant definition to use #[serde(default)] constant: Option, + /// Raw-layout only: byte offset of this field within the account + #[serde(default)] + offset: Option, + /// Raw-layout only: how this field's bytes are produced + #[serde(default)] + encoding: Option, }, } @@ -863,6 +903,8 @@ impl From for Property { label, description, constant, + offset, + encoding, } => { let kind = match kind.as_deref() { Some("constant_ref") => PropertyKind::ConstantRef, @@ -874,6 +916,8 @@ impl From for Property { label, description, constant, + offset, + encoding, } } } @@ -924,14 +968,18 @@ pub struct YamlOverrideTemplateCollection { /// Account type name from the IDL (optional, can be overridden per template) #[serde(default)] pub account_type: Option, - /// Path to shared IDL file - pub idl_file_path: String, + /// Path to shared IDL file. Absent for programs that publish no IDL. + #[serde(default)] + pub idl_file_path: Option, /// Common tags for all templates #[serde(default)] pub tags: Vec, /// Protocol-specific constants shared by all templates in this collection #[serde(default)] pub constants: HashMap, + /// Byte layout, for programs with no usable IDL. Shared by every template in the collection. + #[serde(default)] + pub raw_layout: Option, /// The templates pub templates: Vec, } @@ -954,6 +1002,242 @@ pub struct YamlOverrideTemplateEntry { pub llm_context: Option, } +// ======================================== +// Raw byte layouts (programs with no usable IDL) +// ======================================== + +/// How a raw-layout field's bytes are produced. Every variant is integer-exact: values arrive as +/// JSON integers or decimal strings and are written little-endian, never routed through f64. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS))] +pub enum RawEncoding { + U8, + U16, + U32, + U64, + U128, + I32, + I64, + I128, + /// A signed 32-bit value written to `count` slots, `stride` bytes apart. + /// + /// Exists because some layouts repeat one logical setting across a run of fixed-size records, and + /// exposing one property per record means exposing several that must agree - a worse footgun than + /// whatever it was meant to fix, such as a quote ladder repeated across fixed-size records. + I32Strided { + count: usize, + stride: usize, + }, + /// A base58 pubkey, written as 32 bytes. + Bytes32, + /// The slot the override materializes at, plus `lead` (may be negative). + Slot { + lead: i64, + }, +} + +impl RawEncoding { + /// Byte width of this encoding. + pub fn width(&self) -> usize { + match self { + RawEncoding::U8 => 1, + RawEncoding::U16 => 2, + RawEncoding::U32 | RawEncoding::I32 | RawEncoding::I32Strided { .. } => 4, + RawEncoding::U64 | RawEncoding::I64 | RawEncoding::Slot { .. } => 8, + RawEncoding::U128 | RawEncoding::I128 => 16, + RawEncoding::Bytes32 => 32, + } + } + + /// How many times the encoded value is written, and the byte step between writes. + /// + /// Every scalar writes once. Returning this uniformly lets `materialize` place strided and scalar + /// encodings with the same loop instead of special-casing one of them. + pub fn placements(&self) -> (usize, usize) { + match self { + RawEncoding::I32Strided { count, stride } => (*count, *stride), + other => (1, other.width()), + } + } + + /// The little-endian bytes for `value`. `target_slot` is only read by [`RawEncoding::Slot`]. + pub fn encode(&self, value: &serde_json::Value, target_slot: Slot) -> Result, String> { + // Read the digits as text so nothing passes through f64, which cannot hold a u128 + // exactly. A decimal string is the only way to express values above u64::MAX in JSON. + let digits = |what: &str| -> Result { + match value { + serde_json::Value::Number(n) if n.as_u64().is_none() && n.as_i64().is_none() => { + Err(format!( + "{n} exceeds what a JSON number can hold exactly; pass this {what} as a \ + decimal string instead" + )) + } + serde_json::Value::Number(n) => Ok(n.to_string()), + serde_json::Value::String(s) => Ok(s.trim().to_string()), + other => Err(format!( + "expected a number or decimal string for {what}, found {other}" + )), + } + }; + macro_rules! int { + ($ty:ty, $what:expr) => {{ + let d = digits($what)?; + d.parse::<$ty>() + .map_err(|e| format!("invalid {}: '{d}': {e}", $what))? + .to_le_bytes() + .to_vec() + }}; + } + Ok(match self { + RawEncoding::U8 => int!(u8, "u8"), + RawEncoding::U16 => int!(u16, "u16"), + RawEncoding::U32 => int!(u32, "u32"), + RawEncoding::U64 => int!(u64, "u64"), + RawEncoding::U128 => int!(u128, "u128"), + RawEncoding::I32 | RawEncoding::I32Strided { .. } => int!(i32, "i32"), + RawEncoding::I64 => int!(i64, "i64"), + RawEncoding::I128 => int!(i128, "i128"), + RawEncoding::Bytes32 => { + let text = value + .as_str() + .ok_or_else(|| "expected a base58 pubkey string".to_string())?; + Pubkey::from_str(text) + .map_err(|e| format!("invalid pubkey '{text}': {e}"))? + .to_bytes() + .to_vec() + } + RawEncoding::Slot { lead } => { + let lead = match value { + serde_json::Value::Null => *lead, + _ => { + let d = digits("slot lead")?; + d.parse::() + .map_err(|e| format!("invalid slot lead: '{d}': {e}"))? + } + }; + let slot = if lead >= 0 { + target_slot.checked_add(lead as u64).ok_or_else(|| { + format!("slot {target_slot} plus lead {lead} exceeds u64::MAX") + })? + } else { + target_slot.checked_sub(lead.unsigned_abs()).unwrap_or(0) + }; + slot.to_le_bytes().to_vec() + } + }) + } +} + +/// Bytes that must be present for an account to be the one a raw layout describes. Without an +/// IDL there is no discriminator to resolve the type, so this is the only thing standing between +/// a raw write and silently corrupting an unrelated account. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS))] +pub struct RawMagic { + pub offset: usize, + /// Expected bytes, as an ASCII string or a byte list. + pub bytes: Vec, +} + +/// A byte-level description of an account, used instead of an IDL. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "camelCase")] +// Deliberately no `ts(export)`: override templates are not part of the TS surface, so the three +// raw-layout types have nothing referencing them there and exporting them produced no file. +#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS))] +pub struct RawLayout { + /// Exact account size. A mismatch means this is not the account the layout describes. + /// Serialized camelCase for the JSON API; the alias keeps the YAML snake_case like its peers. + #[serde(alias = "account_size")] + #[cfg_attr(feature = "ts-bindings", ts(type = "number"))] + pub account_size: usize, + /// Optional type tag. Omit for programs that have none. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub magic: Option, +} + +impl RawLayout { + /// Rejects an account that is not the shape this layout describes. + pub fn guard(&self, data: &[u8]) -> Result<(), String> { + if data.len() != self.account_size { + return Err(format!( + "account is {} bytes, the layout describes {}", + data.len(), + self.account_size + )); + } + if let Some(magic) = &self.magic { + let end = magic + .offset + .checked_add(magic.bytes.len()) + .ok_or_else(|| "magic offset overflow".to_string())?; + if end > data.len() || &data[magic.offset..end] != magic.bytes.as_slice() { + return Err(format!( + "magic bytes at offset {} do not match; this is not the expected account", + magic.offset + )); + } + } + Ok(()) + } + + /// Writes `values` into a copy of `data` using each property's offset and encoding. + pub fn materialize( + &self, + data: &[u8], + properties: &[Property], + values: &HashMap, + target_slot: Slot, + ) -> Result, String> { + self.guard(data)?; + let mut out = data.to_vec(); + for (name, value) in values { + let property = properties + .iter() + .find(|p| &p.path == name) + .ok_or_else(|| format!("'{name}' is not a property of this raw-layout template"))?; + let (Some(offset), Some(encoding)) = (property.offset, property.encoding.as_ref()) + else { + return Err(format!("property '{name}' has no offset or encoding")); + }; + let bytes = encoding.encode(value, target_slot)?; + let (count, stride) = encoding.placements(); + for i in 0..count { + let at = offset + .checked_add( + i.checked_mul(stride) + .ok_or_else(|| format!("stride overflow for '{name}'"))?, + ) + .ok_or_else(|| format!("offset overflow for '{name}'"))?; + let end = at + .checked_add(bytes.len()) + .ok_or_else(|| format!("offset overflow for '{name}'"))?; + if end > out.len() { + // Scalars keep the original wording; only a strided run needs to explain itself. + return Err(if count == 1 { + format!( + "'{name}' at offset {offset} + {} bytes exceeds the {} byte account", + bytes.len(), + out.len() + ) + } else { + format!( + "'{name}' writes {count} x {} bytes from offset {offset} every \ + {stride}, which exceeds the {} byte account", + bytes.len(), + out.len() + ) + }); + } + out[at..end].copy_from_slice(&bytes); + } + } + Ok(out) + } +} + /// Walks a dot-notation path: struct fields by name, array elements by index. /// /// Returns the last named field and the type at the path's end. They differ on a trailing index: @@ -1050,7 +1334,7 @@ fn idl_field_docs(idl: &Idl, account_type: &str, path: &str) -> Option { /// supply one, so field guidance is not written twice. fn describe_properties_from_idl( properties: Vec, - idl: &Idl, + idl: Option<&Idl>, account_type: &str, ) -> Vec { properties @@ -1058,7 +1342,10 @@ fn describe_properties_from_idl( .map(|yaml| { let mut property: Property = yaml.into(); if property.description.is_none() { - property.description = idl_field_docs(idl, account_type, &property.path); + // Only a fallback. A raw_layout collection with no IDL must spell out every + // description in the YAML, since there is no schema to borrow docs from. + property.description = + idl.and_then(|idl| idl_field_docs(idl, account_type, &property.path)); } property }) @@ -1067,7 +1354,7 @@ fn describe_properties_from_idl( impl YamlOverrideTemplateCollection { /// Convert collection to runtime OverrideTemplates with loaded IDL - pub fn to_override_templates(self, idl: Idl) -> Vec { + pub fn to_override_templates(self, idl: Option) -> Vec { // Convert constants once for sharing let constants: HashMap = self .constants @@ -1090,11 +1377,16 @@ impl YamlOverrideTemplateCollection { protocol: self.protocol.clone(), idl: idl.clone(), address: entry.address.into(), - properties: describe_properties_from_idl(entry.properties, &idl, &account_type), + properties: describe_properties_from_idl( + entry.properties, + idl.as_ref(), + &account_type, + ), account_type, constants: constants.clone(), tags: self.tags.clone(), llm_context: entry.llm_context, + raw_layout: self.raw_layout.clone(), } }) .collect() @@ -1132,7 +1424,7 @@ impl YamlOverrideTemplate { name: self.name, description: self.description, protocol: self.protocol, - idl: self.idl, + idl: Some(self.idl), address: self.address.into(), account_type: self.account_type, properties: self.properties.into_iter().map(Into::into).collect(), @@ -1143,6 +1435,7 @@ impl YamlOverrideTemplate { .collect(), tags: self.tags, llm_context: self.llm_context, + raw_layout: None, } } } @@ -1241,6 +1534,169 @@ mod tests { use super::PdaSeed; + /// The encoding layer must never route a value through f64: a 2^88-scaled price is a 29-digit + /// integer and f64 carries about 16 significant digits. + #[test] + fn raw_encoding_writes_large_values_exactly() { + use super::RawEncoding; + + let huge: u128 = 50u128 * (1u128 << 88); + let bytes = RawEncoding::U128 + .encode(&json!(huge.to_string()), 0) + .expect("decimal string"); + assert_eq!(u128::from_le_bytes(bytes.try_into().unwrap()), huge); + + // A bare JSON number that big has already lost digits, so it must be refused rather than + // silently written wrong. + let err = RawEncoding::U128 + .encode(&json!(1.152921504606847e21), 0) + .expect_err("an inexact JSON number must be refused"); + assert!(err.contains("decimal string"), "unexpected error: {err}"); + } + + #[test] + fn raw_encoding_handles_signed_and_slot_fields() { + use super::RawEncoding; + + let bytes = RawEncoding::I64.encode(&json!(-25599i64 << 32), 0).unwrap(); + assert_eq!(i64::from_le_bytes(bytes.try_into().unwrap()) >> 32, -25599); + + // The supplied value is the lead, so one property covers live and stale. + let bytes = RawEncoding::Slot { lead: 0 } + .encode(&json!(0), 500) + .unwrap(); + assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), 500); + + let bytes = RawEncoding::Slot { lead: 0 } + .encode(&json!(-5), 500) + .unwrap(); + assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), 495); + + // The manifest lead is the default, used when no value is given. + let bytes = RawEncoding::Slot { lead: -1 } + .encode(&json!(null), 500) + .unwrap(); + assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), 499); + + // A lead that would go below zero clamps rather than wrapping. + let bytes = RawEncoding::Slot { lead: 0 } + .encode(&json!(-10), 3) + .unwrap(); + assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), 0); + + // Slot is a u64. Values above i64::MAX must not wrap through a signed cast and become zero. + let large_slot = i64::MAX as u64 + 1; + let bytes = RawEncoding::Slot { lead: 0 } + .encode(&json!(0), large_slot) + .unwrap(); + assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), large_slot); + + let bytes = RawEncoding::Slot { lead: 0 } + .encode(&json!(-1), u64::MAX) + .unwrap(); + assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), u64::MAX - 1); + + let bytes = RawEncoding::Slot { lead: 0 } + .encode(&json!(0), u64::MAX) + .unwrap(); + assert_eq!(u64::from_le_bytes(bytes.try_into().unwrap()), u64::MAX); + + let err = RawEncoding::Slot { lead: 0 } + .encode(&json!(1), u64::MAX) + .expect_err("a positive lead must not wrap past u64::MAX"); + assert!(err.contains("exceeds u64::MAX"), "unexpected error: {err}"); + } + + #[test] + fn raw_layout_rejects_writes_past_the_end_of_the_account() { + use super::{Property, RawEncoding, RawLayout}; + + let layout = RawLayout { + account_size: 16, + magic: None, + }; + let mut property = Property::field("tail".to_string()); + property.offset = Some(12); + property.encoding = Some(RawEncoding::U64); + + let err = layout + .materialize( + &[0u8; 16], + &[property], + &HashMap::from([("tail".to_string(), json!(1))]), + 0, + ) + .expect_err("a field crossing the end must be refused"); + assert!(err.contains("exceeds"), "unexpected error: {err}"); + } + + #[test] + fn i32_strided_writes_every_slot_and_nothing_between() { + use super::{Property, RawEncoding, RawLayout}; + let layout = RawLayout { + account_size: 64, + magic: None, + }; + let mut property = Property::field("ticks".to_string()); + property.offset = Some(4); + property.encoding = Some(RawEncoding::I32Strided { + count: 3, + stride: 16, + }); + + let out = layout + .materialize( + &[0u8; 64], + &[property], + &HashMap::from([("ticks".to_string(), json!(-25_600))]), + 0, + ) + .expect("strided write"); + + for i in 0..3usize { + let at = 4 + i * 16; + assert_eq!( + i32::from_le_bytes(out[at..at + 4].try_into().unwrap()), + -25_600, + "slot {i} at offset {at} should carry the value" + ); + } + // Everything outside the three four-byte spans must be untouched. + let written: Vec = (0..3).flat_map(|i| (4 + i * 16)..(8 + i * 16)).collect(); + for (i, b) in out.iter().enumerate() { + if !written.contains(&i) { + assert_eq!( + *b, 0, + "byte {i} lies between strided slots and must not change" + ); + } + } + } + + #[test] + fn i32_strided_rejects_a_run_that_leaves_the_account() { + use super::{Property, RawEncoding, RawLayout}; + let layout = RawLayout { + account_size: 32, + magic: None, + }; + let mut property = Property::field("ticks".to_string()); + property.offset = Some(4); + property.encoding = Some(RawEncoding::I32Strided { + count: 3, + stride: 16, + }); + let err = layout + .materialize( + &[0u8; 32], + &[property], + &HashMap::from([("ticks".to_string(), json!(1))]), + 0, + ) + .expect_err("a run crossing the end must be refused"); + assert!(err.contains("exceeds"), "unexpected error: {err}"); + } + #[test] fn u16_be_ref_rejects_out_of_range_values() { let seed = PdaSeed::U16BeRef("index".to_string()); From 32e310c2aa3e08897677791030547bb3c6f15af2 Mon Sep 17 00:00:00 2001 From: bakasura980 Date: Mon, 14 Sep 2026 16:05:33 +0300 Subject: [PATCH 2/2] fix(scenarios): validate raw-layout templates on registration --- crates/core/src/scenarios/registry.rs | 138 ++++++++++++++++++++++++++ crates/types/src/scenarios.rs | 68 +++++++++++++ 2 files changed, 206 insertions(+) diff --git a/crates/core/src/scenarios/registry.rs b/crates/core/src/scenarios/registry.rs index 11f419e19..a421a3f91 100644 --- a/crates/core/src/scenarios/registry.rs +++ b/crates/core/src/scenarios/registry.rs @@ -204,6 +204,7 @@ impl TemplateRegistry { overrides_content: &str, protocol_name: &str, ) { + let requires_raw_layout = idl.is_none(); let collection = match serde_yaml::from_str::(overrides_content) { Ok(c) => c, @@ -213,6 +214,27 @@ impl TemplateRegistry { // Convert all templates in the collection let templates = collection.to_override_templates(idl); + // Validate the entire collection before mutating the registry, so one malformed entry + // cannot leave its valid siblings partially registered. + if requires_raw_layout { + for template in &templates { + let layout = template.raw_layout.as_ref().unwrap_or_else(|| { + panic!( + "unable to load {protocol_name} overrides: raw-layout template '{}' has no raw_layout", + template.id + ) + }); + layout + .validate_properties(&template.properties) + .unwrap_or_else(|e| { + panic!( + "unable to load {protocol_name} overrides: invalid raw-layout template '{}': {e}", + template.id + ) + }); + } + } + // Register each template for template in templates { let template_id = template.id.clone(); @@ -614,6 +636,122 @@ templates: assert_eq!(u64::from_le_bytes(output[8..16].try_into().unwrap()), 42); } + #[test] + fn raw_layout_collection_rejects_invalid_write_definitions_at_load_time() { + fn rejected(yaml: &str, expected: &str) { + let result = std::panic::catch_unwind(|| { + let mut registry = TemplateRegistry::default(); + registry.load_raw_layout_overrides(yaml, "broken"); + }); + let panic = result.expect_err("invalid raw-layout collection must be rejected"); + let message = panic + .downcast_ref::() + .map(String::as_str) + .or_else(|| panic.downcast_ref::<&str>().copied()) + .expect("panic message"); + assert!( + message.contains(expected), + "expected {expected:?} in {message:?}" + ); + } + + rejected( + r#" +protocol: Broken +version: v1 +templates: + - id: no-layout + name: No layout + description: Invalid + address: { type: pubkey, value: "11111111111111111111111111111111" } + properties: [] +"#, + "has no raw_layout", + ); + + rejected( + r#" +protocol: Broken +version: v1 +raw_layout: { account_size: 16 } +templates: + - id: no-offset + name: No offset + description: Invalid + address: { type: pubkey, value: "11111111111111111111111111111111" } + properties: + - { path: value, encoding: u64 } +"#, + "missing an offset", + ); + + rejected( + r#" +protocol: Broken +version: v1 +raw_layout: { account_size: 16 } +templates: + - id: no-encoding + name: No encoding + description: Invalid + address: { type: pubkey, value: "11111111111111111111111111111111" } + properties: + - { path: value, offset: 8 } +"#, + "missing an encoding", + ); + + rejected( + r#" +protocol: Broken +version: v1 +raw_layout: { account_size: 16 } +templates: + - id: out-of-bounds + name: Out of bounds + description: Invalid + address: { type: pubkey, value: "11111111111111111111111111111111" } + properties: + - { path: value, offset: 12, encoding: u64 } +"#, + "beyond the 16 byte account", + ); + + let mut registry = TemplateRegistry::default(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + registry.load_raw_layout_overrides( + r#" +protocol: Broken +version: v1 +raw_layout: { account_size: 16 } +templates: + - id: valid-sibling + name: Valid sibling + description: Valid alone + address: { type: pubkey, value: "11111111111111111111111111111111" } + properties: + - { path: value, offset: 8, encoding: u64 } + - id: invalid-sibling + name: Invalid sibling + description: Invalid + address: { type: pubkey, value: "11111111111111111111111111111111" } + properties: + - { path: value, encoding: u64 } +"#, + "broken", + ); + })); + assert!( + result.is_err(), + "invalid sibling must reject the collection" + ); + assert_eq!( + registry.count(), + 0, + "validation must finish before any sibling is registered" + ); + } + #[test] fn test_jupiter_template_loads_correctly() { let registry = TemplateRegistry::new(); diff --git a/crates/types/src/scenarios.rs b/crates/types/src/scenarios.rs index 0e25a7a4a..fab64d27b 100644 --- a/crates/types/src/scenarios.rs +++ b/crates/types/src/scenarios.rs @@ -1159,6 +1159,74 @@ pub struct RawLayout { } impl RawLayout { + /// Validates every byte range a template can write before the template enters the registry. + pub fn validate_properties(&self, properties: &[Property]) -> Result<(), String> { + if self.account_size == 0 { + return Err("raw_layout account_size must be greater than zero".to_string()); + } + + if let Some(magic) = &self.magic { + let end = magic + .offset + .checked_add(magic.bytes.len()) + .ok_or_else(|| "raw_layout magic offset overflow".to_string())?; + if end > self.account_size { + return Err(format!( + "raw_layout magic at offset {} + {} bytes exceeds the {} byte account", + magic.offset, + magic.bytes.len(), + self.account_size + )); + } + } + + for property in properties { + // Constant references select PDA seeds or catalog values; they are not account writes. + if property.is_constant_ref() { + continue; + } + + let offset = property.offset.ok_or_else(|| { + format!( + "writable raw-layout property '{}' is missing an offset", + property.path + ) + })?; + let encoding = property.encoding.as_ref().ok_or_else(|| { + format!( + "writable raw-layout property '{}' is missing an encoding", + property.path + ) + })?; + let (count, stride) = encoding.placements(); + if count == 0 { + return Err(format!( + "writable raw-layout property '{}' has zero placements", + property.path + )); + } + + let final_offset = offset + .checked_add( + (count - 1) + .checked_mul(stride) + .ok_or_else(|| format!("stride overflow for '{}'", property.path))?, + ) + .ok_or_else(|| format!("offset overflow for '{}'", property.path))?; + let end = final_offset + .checked_add(encoding.width()) + .ok_or_else(|| format!("offset overflow for '{}'", property.path))?; + if end > self.account_size { + return Err(format!( + "writable raw-layout property '{}' ends at byte {}, beyond the {} byte account", + property.path, end, self.account_size + )); + } + } + + Ok(()) + } + /// Rejects an account that is not the shape this layout describes. pub fn guard(&self, data: &[u8]) -> Result<(), String> { if data.len() != self.account_size {