feat(scenarios): add reusable raw-layout overrides

crates/core/src/scenarios/README.md

@@ -56,6 +56,29 @@ cargo test -p surfpool-core --features integration-tests kamino
 Set `SURFPOOL_TEST_RPC_URL` to use a private endpoint instead of the public one. The default test
 run needs no network.
 
+### Programs with no IDL
+
+Programs that publish no usable IDL can describe their account bytes directly in an override
+collection:
+
+```yaml
+raw_layout:
+  account_size: 128
+  magic: { offset: 0, bytes: [69, 88, 65, 77, 80, 76, 69] }
+
+templates:
+  - id: example-market-price
+    properties:
+      - path: price
+        offset: 32
+        encoding: u128
+```
+
+Raw-layout overrides use the same scenario API as IDL-backed templates. Values are encoded as
+little-endian integers and can be written once or repeatedly at a fixed stride. Account-size and
+magic guards ensure a template refuses the wrong account before changing any bytes. Large integers
+that exceed `u64` should be supplied as decimal strings so JSON parsing cannot lose precision.
+
 ### Override Templates
 Directly using the `surfnet_registerScenario` endpoint requires building out a map of account keys that are specific to the schema of the account that is being written to.
 This is a cumbersome process in most cases.

crates/core/src/scenarios/registry.rs

@@ -189,7 +189,22 @@ impl TemplateRegistry {
             Ok(idl) => idl,
             Err(e) => panic!("unable to load {} idl: {}", protocol_name, e),
         };
+        self.load_collection(Some(idl), overrides_content, protocol_name);
+    }
+
+    /// For programs that publish no IDL. Their templates must carry a `raw_layout` and spell out
+    /// every property description, since there is no schema to fall back on.
+    pub fn load_raw_layout_overrides(&mut self, overrides_content: &str, protocol_name: &str) {
+        self.load_collection(None, overrides_content, protocol_name);
+    }
 
+    fn load_collection(
+        &mut self,
+        idl: Option<anchor_lang_idl::types::Idl>,
+        overrides_content: &str,
+        protocol_name: &str,
+    ) {
+        let requires_raw_layout = idl.is_none();
         let collection =
             match serde_yaml::from_str::<YamlOverrideTemplateCollection>(overrides_content) {
                 Ok(c) => c,
@@ -199,6 +214,27 @@ impl TemplateRegistry {
         // Convert all templates in the collection
         let templates = collection.to_override_templates(idl);
 
+        // Validate the entire collection before mutating the registry, so one malformed entry
+        // cannot leave its valid siblings partially registered.
+        if requires_raw_layout {
+            for template in &templates {
+                let layout = template.raw_layout.as_ref().unwrap_or_else(|| {
+                    panic!(
+                        "unable to load {protocol_name} overrides: raw-layout template '{}' has no raw_layout",
+                        template.id
+                    )
+                });
+                layout
+                    .validate_properties(&template.properties)
+                    .unwrap_or_else(|e| {
+                        panic!(
+                            "unable to load {protocol_name} overrides: invalid raw-layout template '{}': {e}",
+                            template.id
+                        )
+                    });
+            }
+        }
+
         // Register each template
         for template in templates {
             let template_id = template.id.clone();
@@ -557,6 +593,165 @@ mod tests {
         assert!(registry.contains("pump-amm-global-config"));
     }
 
+    #[test]
+    fn raw_layout_collection_loads_without_an_idl() {
+        const OVERRIDES: &str = r#"
+protocol: Example
+version: v1
+account_type: State
+raw_layout:
+  account_size: 16
+  magic:
+    offset: 0
+    bytes: [69, 88]
+templates:
+  - id: example-raw-value
+    name: Override Value
+    description: Override one integer in an example binary account
+    address:
+      type: pubkey
+      value: "11111111111111111111111111111111"
+    properties:
+      - path: value
+        offset: 8
+        encoding: u64
+        label: Value
+        description: Example unsigned integer
+"#;
+
+        let mut registry = TemplateRegistry::default();
+        registry.load_raw_layout_overrides(OVERRIDES, "example");
+
+        let template = registry.get("example-raw-value").expect("raw template");
+        assert!(template.idl.is_none());
+        let layout = template.raw_layout.as_ref().expect("raw layout");
+        let output = layout
+            .materialize(
+                &[69, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+                &template.properties,
+                &HashMap::from([("value".to_string(), serde_json::json!(42))]),
+                0,
+            )
+            .expect("materialize raw template");
+        assert_eq!(u64::from_le_bytes(output[8..16].try_into().unwrap()), 42);
+    }
+
+    #[test]
+    fn raw_layout_collection_rejects_invalid_write_definitions_at_load_time() {
+        fn rejected(yaml: &str, expected: &str) {
+            let result = std::panic::catch_unwind(|| {
+                let mut registry = TemplateRegistry::default();
+                registry.load_raw_layout_overrides(yaml, "broken");
+            });
+            let panic = result.expect_err("invalid raw-layout collection must be rejected");
+            let message = panic
+                .downcast_ref::<String>()
+                .map(String::as_str)
+                .or_else(|| panic.downcast_ref::<&str>().copied())
+                .expect("panic message");
+            assert!(
+                message.contains(expected),
+                "expected {expected:?} in {message:?}"
+            );
+        }
+
+        rejected(
+            r#"
+protocol: Broken
+version: v1
+templates:
+  - id: no-layout
+    name: No layout
+    description: Invalid
+    address: { type: pubkey, value: "11111111111111111111111111111111" }
+    properties: []
+"#,
+            "has no raw_layout",
+        );
+
+        rejected(
+            r#"
+protocol: Broken
+version: v1
+raw_layout: { account_size: 16 }
+templates:
+  - id: no-offset
+    name: No offset
+    description: Invalid
+    address: { type: pubkey, value: "11111111111111111111111111111111" }
+    properties:
+      - { path: value, encoding: u64 }
+"#,
+            "missing an offset",
+        );
+
+        rejected(
+            r#"
+protocol: Broken
+version: v1
+raw_layout: { account_size: 16 }
+templates:
+  - id: no-encoding
+    name: No encoding
+    description: Invalid
+    address: { type: pubkey, value: "11111111111111111111111111111111" }
+    properties:
+      - { path: value, offset: 8 }
+"#,
+            "missing an encoding",
+        );
+
+        rejected(
+            r#"
+protocol: Broken
+version: v1
+raw_layout: { account_size: 16 }
+templates:
+  - id: out-of-bounds
+    name: Out of bounds
+    description: Invalid
+    address: { type: pubkey, value: "11111111111111111111111111111111" }
+    properties:
+      - { path: value, offset: 12, encoding: u64 }
+"#,
+            "beyond the 16 byte account",
+        );
+
+        let mut registry = TemplateRegistry::default();
+        let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            registry.load_raw_layout_overrides(
+                r#"
+protocol: Broken
+version: v1
+raw_layout: { account_size: 16 }
+templates:
+  - id: valid-sibling
+    name: Valid sibling
+    description: Valid alone
+    address: { type: pubkey, value: "11111111111111111111111111111111" }
+    properties:
+      - { path: value, offset: 8, encoding: u64 }
+  - id: invalid-sibling
+    name: Invalid sibling
+    description: Invalid
+    address: { type: pubkey, value: "11111111111111111111111111111111" }
+    properties:
+      - { path: value, encoding: u64 }
+"#,
+                "broken",
+            );
+        }));
+        assert!(
+            result.is_err(),
+            "invalid sibling must reject the collection"
+        );
+        assert_eq!(
+            registry.count(),
+            0,
+            "validation must finish before any sibling is registered"
+        );
+    }
+
     #[test]
     fn test_jupiter_template_loads_correctly() {
         let registry = TemplateRegistry::new();
@@ -717,7 +912,7 @@ mod tests {
         let registry = TemplateRegistry::new();
         let jupiter_template = registry.get("jupiter-token-ledger-override").unwrap();
         let has_token_ledger = jupiter_template
-            .idl
+            .idl()
             .accounts
             .iter()
             .any(|acc| acc.name == "TokenLedger");
@@ -1157,18 +1352,24 @@ mod tests {
         let registry = TemplateRegistry::new();
         let mut errors = Vec::new();
 
+        let mut checked = 0usize;
         for template in registry.all() {
+            // Templates for programs that publish no IDL declare their own byte offsets, so there
+            // is no schema for their paths to resolve against. Raw encoding and guarded writes are
+            // covered by the unit tests on `RawLayout`.
+            let Some(idl) = template.idl.as_ref() else {
+                continue;
+            };
             for property in &template.properties {
                 // constant_ref properties are UI dropdowns (e.g. token pickers), not
                 // account fields, so they are not expected to resolve against the IDL.
                 if property.is_constant_ref() {
                     continue;
                 }
-                if let Err(e) = surfpool_types::resolve_idl_type(
-                    &template.idl,
-                    &template.account_type,
-                    &property.path,
-                ) {
+                checked += 1;
+                if let Err(e) =
+                    surfpool_types::resolve_idl_type(idl, &template.account_type, &property.path)
+                {
                     errors.push(format!("[{}] {}: {}", template.id, property.path, e));
                 }
             }
@@ -1180,6 +1381,12 @@ mod tests {
             errors.len(),
             errors.join("\n  ")
         );
+        // Without this the skip above could silently swallow every template and the test would pass
+        // having resolved nothing.
+        assert!(
+            checked > 0,
+            "no property was resolved against an IDL, so this proved nothing"
+        );
     }
 
     #[test]
@@ -1339,7 +1546,7 @@ mod tests {
             ("ref_price.0", IdlType::U16),
         ] {
             let resolved =
-                surfpool_types::resolve_idl_type(&template.idl, &template.account_type, path)
+                surfpool_types::resolve_idl_type(template.idl(), &template.account_type, path)
                     .unwrap_or_else(|e| panic!("{path} should resolve: {e}"));
             assert_eq!(
                 *resolved, expected,
@@ -1352,7 +1559,7 @@ mod tests {
             .get("kamino-obligation-positions")
             .expect("kamino-obligation-positions should exist");
         let resolved = surfpool_types::resolve_idl_type(
-            &obligation.idl,
+            obligation.idl(),
             &obligation.account_type,
             "deposits.0.deposit_reserve",
         )