diff --git a/.gitignore b/.gitignore index 25b6a11c..a8837c6b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,14 @@ skills-lock.json # Engine runtime state (default state_dir from engine.toml). data/ + +# E2E automation: rendered configs with embedded RPC keys + script state +# never get committed. +*.local.toml +scripts/.state +scripts/.env + +# Generated reports under e2e-reports/ (operator commits the filled-in ones +# manually via `git add -f`). +docs/operations/e2e-reports/engine-*.log +docs/operations/e2e-reports/metrics-*.txt diff --git a/docs/operations/e2e-cow-1064-prep.md b/docs/operations/e2e-cow-1064-prep.md new file mode 100644 index 00000000..eb44e4ef --- /dev/null +++ b/docs/operations/e2e-cow-1064-prep.md @@ -0,0 +1,322 @@ +# E2E COW-1064 run-prep punch list + +Companion to `docs/operations/e2e-testnet-runbook.md`. This file +captures every **pinned value** for the 2026-06-18 dry run of the +COW-1064 E2E so the operator can copy-paste through the on-chain +actions without re-deriving any UID, address, or calldata. + +If you are running a *later* COW-1064 (different EOA, different +Safe, different config), do not reuse the UIDs / calldatas — they +are a function of all the pinned config below. Either re-derive +via the Python recipes in this doc, or re-run +`cargo test -p stop-loss --lib cow_1064` to lock the new UID. + +--- + +## 0. Pinned identities (2026-06-18 run) + +| Role | Address | Network | Notes | +|---|---|---|---| +| Test EOA | `0x7bF140727D27ea64b607E042f1225680B40ECa6A` | Sepolia | Bruno-controlled. Funds itself via faucet. | +| Test Safe (single-sig, threshold 1) | `0x14995a1118Caf95833e923faf8Dd155721cd53c2` | Sepolia | EOA is the sole owner. Submits TWAP order. | +| ComposableCoW | `0xfdaFc9d1902f4e0b84f65F49f244b32b31013b74` | Sepolia | Where `create((address,bytes32,bytes),bool)` lands. | +| TWAP handler | `0x6cF1e9cA41f7611dEf408122793c358a3d11E5a5` | Sepolia | `ConditionalOrderParams.handler`. | +| CoWSwapEthFlow | `0xbA3cB449bD2B4ADddBc894D8697F5170800EAdeC` | Sepolia | EthFlow's production deployment; emits `OrderPlacement`. | +| GPv2Settlement | `0x9008D19f58AAbD9eD0D60971565AA8510560ab41` | Sepolia | `setPreSignature(orderUid, signed)` lives here. | +| GPv2VaultRelayer | `0xc92e8bdf79f0507f65a392b0ab4667716bfe0110` | Sepolia | Spender for sell-token ERC-20 approvals. | +| WETH9 | `0xfFf9976782d46CC05630D1f6eBAb18b2324d6B14` | Sepolia | `deposit()` payable wraps ETH; `balanceOf(EOA)` is the sell-side balance. | +| COW Token | `0x0625aFB445C3B6B7B929342a04A22599fd5dBB59` | Sepolia | name="CoW Protocol Token", symbol="COW", decimals=18. | +| GPv2 domain separator | `0xdaee378bd0eb30ddf479272accf91761e697bc00e067a268f95f1d2732ed230b` | Sepolia | EIP-712 domain digest queried from chain. | + +All addresses verified via `eth_getCode > 0` on +`https://ethereum-sepolia-rpc.publicnode.com` as of run prep. + +--- + +## 1. Per-module config pinning + +### stop-loss + +`modules/examples/stop-loss/module.toml` is checked in on the +`feat/e2e-run-config-cow-1064` branch with the production-ready +config for this run. Effective values: + +| Field | Value | Notes | +|---|---|---| +| `oracle_address` | `0x694AA1769357215DE4FAC081bf1f309aDC325306` | Chainlink ETH/USD Sepolia. | +| `decimals` | `8` | Chainlink USD-pair convention. | +| `trigger_price` | `2000.00` | Above the live Sepolia mocked answer (~$1681), `direction=below` → triggers on first block. | +| `owner` | `0x7bF1...Ca6A` | Test EOA. | +| `sell_token` | `0xfFf9...6B14` | WETH9 Sepolia. | +| `buy_token` | `0x0625...BB59` | COW Sepolia. | +| `sell_amount_wei` | `5000000000000000` | 0.005 WETH. | +| `buy_amount_wei` | `20000000000000000000` | 20 COW. Conservative quote at run-prep time. | +| `valid_to_seconds` | `4294967295` | uint32::MAX. | + +### Resulting OrderUid + +The strategy's `build_creation` is pinned by the +`cow_1064_e2e_settings_yield_expected_uid` regression test +(`crates/.../stop-loss/src/strategy.rs`). The canonical UID: + +``` +0xc2b9cb4ea1ee5a86d8049ac09d8f494bf04cca0a68407285f31e2e6379800be87bf140727d27ea64b607e042f1225680b40eca6affffffff +``` + +Decomposition (per `packOrderUidParams`): + +| Offset | Bytes | Field | Value | +|---|---|---|---| +| 0..32 | 32 | `orderDigest` (EIP-712) | `0xc2b9cb4ea1ee5a86d8049ac09d8f494bf04cca0a68407285f31e2e6379800be8` | +| 32..52 | 20 | `owner` | `0x7bf140727d27ea64b607e042f1225680b40eca6a` | +| 52..56 | 4 | `validTo` (uint32) | `0xffffffff` | + +### balance-tracker + +Pinned to the EOA + Safe so the run sees ETH-balance diffs: + +| Field | Value | +|---|---| +| `addresses` | `0x7bF1...Ca6A,0x1499...53c2` | +| `change_threshold` | `1000000000000000` (0.001 ETH) | + +--- + +## 2. On-chain actions for the run window + +> Order: action 1 can be done at any time before/during the run. +> Actions 2-4 should fire **after** the engine prints +> `INFO supervisor ready modules=5 chains=1` so the modules +> observe the events. They are independent; do them in any order. + +### Action 1 (optional, pre-run): wrap 0.01 ETH → 0.01 WETH + +Without WETH, stop-loss will hit `TransferSimulationFailed` -> +`backoff:` write (which is itself a valid terminal-marker per +the COW-1064 acceptance bar). To get the **`submitted:`** path, +wrap first then do action 2. + +- Etherscan: https://sepolia.etherscan.io/address/0xfff9976782d46cc05630d1f6ebab18b2324d6b14#writeContract +- Connect Web3 from the EOA in Metamask +- Function `deposit` → payable value `0.01` ETH → Write + +Verify: `balanceOf(EOA)` returns `10000000000000000` post-tx. + +### Action 2 (optional, only if action 1 done): pre-sign stop-loss order + +- Etherscan: https://sepolia.etherscan.io/address/0x9008d19f58aabd9ed0d60971565aa8510560ab41#writeProxyContract +- Connect Web3 from the EOA +- Function `setPreSignature(bytes orderUid, bool signed)`: + - `orderUid`: + ``` + 0xc2b9cb4ea1ee5a86d8049ac09d8f494bf04cca0a68407285f31e2e6379800be87bf140727d27ea64b607e042f1225680b40eca6affffffff + ``` + - `signed`: `true` +- Write + +Also approve WETH → GPv2VaultRelayer so the settle path is real: + +- Etherscan: https://sepolia.etherscan.io/address/0xfff9976782d46cc05630d1f6ebab18b2324d6b14#writeContract +- Function `approve(address guy, uint256 wad)`: + - `guy`: `0xc92e8bdf79f0507f65a392b0ab4667716bfe0110` + - `wad`: `5000000000000000` (0.005 WETH — matches the order's sell_amount) +- Write + +### Action 3: TWAP conditional order via Safe TX Builder + +Triggers `ConditionalOrderCreated` → twap-monitor writes +`watch:{orderHash}`. The Safe pays the gas (~0.003 ETH); the +order will TRY to settle later but the Safe holds no WETH so +settlement will fail. **That's fine** — only the `create()` +event is required for the acceptance marker. + +- Safe app: https://app.safe.global/transactions/queue?safe=sep:0x14995a1118Caf95833e923faf8Dd155721cd53c2 +- New transaction → Transaction Builder +- Enter contract address: `0xfdaFc9d1902f4e0b84f65F49f244b32b31013b74` +- Toggle "Use custom data (hex encoded)" ON +- Custom data: + +``` +0x6bfae1ca000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000010000000000000000000000006cf1e9ca41f7611def408122793c358a3d11e5a5000000000000000000000000000000000000000000000000000000006670f00000000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000140000000000000000000000000fff9976782d46cc05630d1f6ebab18b2324d6b140000000000000000000000000625afb445c3b6b7b929342a04a22599fd5dbb5900000000000000000000000014995a1118caf95833e923faf8dd155721cd53c200000000000000000000000000000000000000000000000000038d7ea4c6800000000000000000000000000000000000000000000000000006f05b59d3b2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000025800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +``` + +(516 bytes — the `create(ConditionalOrderParams, bool dispatch)` +call with a 2-part TWAP from WETH → COW, 0.001 WETH per part, +600 s between parts, salt pinned to `0x...6670f000`.) + +- ETH value: `0` +- Create batch → Send batch → sign with the EOA + +Expected log within 1-2 Sepolia blocks: + +``` +INFO twap-monitor watch:0x chain_id=11155111 +``` + +### Action 4: EthFlow swap via cow-swap UI + +Triggers `OrderPlacement` → ethflow-watcher writes +`submitted:{uid}` (or `dropped:{uid}` if the orderbook rejects; +both are valid terminal markers). + +Easiest path is the cow-swap UI: + +1. https://swap.cow.fi/#/11155111/swap/ETH/COW (Sepolia) +2. Connect Metamask, EOA selected, network=Sepolia +3. Sell amount: `0.005` ETH +4. Click "Swap" → it builds the EthFlow `createOrder` tx +5. Approve in Metamask + +The UI handles `quoteId` resolution + `appData` IPFS pinning + +EthFlow contract call. Sell amount is small enough to fit in the +~0.05 ETH budget plus gas. + +Expected log within 1-2 Sepolia blocks: + +``` +INFO ethflow-watcher submitted:0x +``` + +If the UI errors out (Sepolia orderbook can be flaky), fallback +to calling EthFlow directly via Etherscan: + +- https://sepolia.etherscan.io/address/0xba3cb449bd2b4adddbc894d8697f5170800eadec#writeContract +- Function `createOrder((address,address,uint256,uint256,bytes32,uint256,uint32,bool,int64))` +- The shape of the tuple needs the orderbook quote endpoint hit + first to get `feeAmount` + `quoteId` — easier to defer to the + UI for the run. + +--- + +## 3. Validation snippets for the operator + +Run these in a separate shell while the engine is up: + +```bash +RPC="wss://eth-sepolia.g.alchemy.com/v2/" # replace +EOA="0x7bF140727D27ea64b607E042f1225680B40ECa6A" +SAFE="0x14995a1118Caf95833e923faf8Dd155721cd53c2" +WETH="0xfFf9976782d46CC05630D1f6eBAb18b2324d6B14" + +# EOA + Safe balances +cast balance $EOA --rpc-url $RPC +cast balance $SAFE --rpc-url $RPC + +# EOA WETH balance + GPv2VaultRelayer allowance +cast call $WETH "balanceOf(address)(uint256)" $EOA --rpc-url $RPC +cast call $WETH "allowance(address,address)(uint256)" \ + $EOA 0xc92e8bdf79f0507f65a392b0ab4667716bfe0110 --rpc-url $RPC + +# Did setPreSignature land? +cast call 0x9008D19f58AAbD9eD0D60971565AA8510560ab41 \ + "preSignature(bytes)(uint256)" \ + 0xc2b9cb4ea1ee5a86d8049ac09d8f494bf04cca0a68407285f31e2e6379800be87bf140727d27ea64b607e042f1225680b40eca6affffffff \ + --rpc-url $RPC +# Returns 1 if pre-signed, 0 otherwise. + +# Mine the supervisor log for terminal markers in real time +journalctl -u shepherd -f --output=json \ + | jq -r '.MESSAGE | fromjson? | select(.fields.message | test("watch:|submitted:|dropped:|backoff:|TRIGGERED")) | "\(.fields.module): \(.fields.message)"' +``` + +(If you don't have `cast` installed: `curl -L https://foundry.paradigm.xyz | bash && foundryup`.) + +--- + +## 4. Recipes for re-deriving the pinned values + +If anything in section 0 drifts, regenerate from these recipes. + +### 4.1 OrderUid + +Either: + +```bash +cargo test -p stop-loss --lib cow_1064 -- --nocapture +``` + +(asserts against the same constants pinned in `module.toml`, +fails loudly if the EIP-712 type-hash or domain separator +shifts). + +Or with raw Python: + +```python +from eth_utils import keccak + +# Replace these 8 values to re-derive +DOMAIN_SEP = bytes.fromhex("daee378bd0eb30ddf479272accf91761e697bc00e067a268f95f1d2732ed230b") +SELL_TOKEN = bytes.fromhex("fFf9976782d46CC05630D1f6eBAb18b2324d6B14") +BUY_TOKEN = bytes.fromhex("0625aFB445C3B6B7B929342a04A22599fd5dBB59") +OWNER = bytes.fromhex("7bF140727D27ea64b607E042f1225680B40ECa6A") +RECEIVER = OWNER +SELL_AMOUNT = 5_000_000_000_000_000 +BUY_AMOUNT = 20_000_000_000_000_000_000 +VALID_TO = 4_294_967_295 + +APP_DATA = bytes.fromhex("b48d38f93eaa084033fc5970bf96e559c33c4cdc07d889ab00b4d63f9590739d") # keccak("{}") +KIND_SELL = keccak(b"sell") +ERC20 = keccak(b"erc20") +TYPE_HASH = keccak(b"Order(address sellToken,address buyToken,address receiver,uint256 sellAmount,uint256 buyAmount,uint32 validTo,bytes32 appData,uint256 feeAmount,string kind,bool partiallyFillable,string sellTokenBalance,string buyTokenBalance)") +pad32 = lambda b: bytes(32-len(b)) + b +uint = lambda v: v.to_bytes(32, "big") +struct_hash = keccak( + TYPE_HASH + pad32(SELL_TOKEN) + pad32(BUY_TOKEN) + pad32(RECEIVER) + + uint(SELL_AMOUNT) + uint(BUY_AMOUNT) + uint(VALID_TO) + + APP_DATA + uint(0) + KIND_SELL + + b"\x00"*32 + ERC20 + ERC20 # partiallyFillable=false +) +order_digest = keccak(b"\x19\x01" + DOMAIN_SEP + struct_hash) +uid = order_digest + OWNER + VALID_TO.to_bytes(4, "big") +print("0x" + uid.hex()) +``` + +### 4.2 ComposableCoW.create() calldata + +```python +from eth_utils import keccak +from eth_abi import encode + +selector = keccak(b"create((address,bytes32,bytes),bool)")[:4] +# Edit these 10 fields to retarget the TWAP +static = encode( + ["(address,address,address,uint256,uint256,uint256,uint256,uint256,uint256,bytes32)"], + [( + "0xfFf9976782d46CC05630D1f6eBAb18b2324d6B14", # sellToken + "0x0625aFB445C3B6B7B929342a04A22599fd5dBB59", # buyToken + "0x14995a1118Caf95833e923faf8Dd155721cd53c2", # receiver + 1_000_000_000_000_000, 500_000_000_000_000_000, # partSellAmount, minPartLimit + 0, 2, 600, 0, # t0, n, t, span + b"\x00" * 32, # appData + )] +) +calldata = selector + encode( + ["(address,bytes32,bytes)", "bool"], + [( + "0x6cF1e9cA41f7611dEf408122793c358a3d11E5a5", # TWAP handler + bytes.fromhex("000000000000000000000000000000000000000000000000000000006670f000"), # salt + static, + ), True] +) +print("0x" + calldata.hex()) +``` + +--- + +## 5. Acceptance checklist for THIS run + +Hand-check at the end of the run (also goes in +`e2e-report-YYYY-MM-DD.md` section 7): + +- [ ] EOA at `0x7bF1...Ca6A` still has ≥ 0.03 ETH remaining +- [ ] twap-monitor logged `watch:0x...` after action 3 +- [ ] ethflow-watcher logged `submitted:0x...` after action 4 +- [ ] stop-loss logged `backoff:` or `TRIGGERED + submitted:` (depending on whether action 1+2 ran) +- [ ] price-alert logged `TRIGGERED` on first block +- [ ] balance-tracker logged a `last:0x7bf1...` write on first block + at least one Warn diff log over the run window +- [ ] `shepherd_module_poisoned{...} == 0` for all 5 modules at end +- [ ] `shepherd_module_errors_total{error_kind="trap"} == 0` for all modules +- [ ] ≥ 1500 Sepolia blocks dispatched (`block delta` in report section 2) + +If all green: COW-1064 closes, COW-1031 7-day soak can start +on the same code. diff --git a/modules/examples/balance-tracker/module.toml b/modules/examples/balance-tracker/module.toml index 2f4bd17f..a81a855b 100644 --- a/modules/examples/balance-tracker/module.toml +++ b/modules/examples/balance-tracker/module.toml @@ -27,6 +27,16 @@ chain_id = 11155111 [config] # Comma-separated list of 0x-prefixed 20-byte addresses. Whitespace # around entries is tolerated. -addresses = "0x70997970C51812dc3A010C7d01b50e0d17dc79C8,0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" -# Change threshold in wei. Default is 0.1 ETH = 10**17. -change_threshold = "100000000000000000" +# +# COW-1064 E2E pinning: the test EOA + Safe co-located on Sepolia. +# Both fund themselves during the runbook's prep step (EOA via +# faucet, Safe via EOA send), so balance-tracker writes +# `last:0x7bf1...` and `last:0x1499...` on the first dispatch and +# logs a Warn diff if the Safe later receives the TWAP order's +# gas-side transfer (or any subsequent move >= change_threshold). +addresses = "0x7bF140727D27ea64b607E042f1225680B40ECa6A,0x14995a1118Caf95833e923faf8Dd155721cd53c2" +# Change threshold in wei. Lowered from 0.1 ETH to 0.001 ETH so the +# E2E run's small on-chain actions (wrap-to-WETH ~0.005 ETH gas cost, +# TWAP Safe call ~0.003 ETH gas) show as diffs in the Warn log; +# production deploys leave this at 0.1 ETH or higher. +change_threshold = "1000000000000000" diff --git a/modules/examples/stop-loss/module.toml b/modules/examples/stop-loss/module.toml index 17cebad5..9d43bd23 100644 --- a/modules/examples/stop-loss/module.toml +++ b/modules/examples/stop-loss/module.toml @@ -28,14 +28,34 @@ chain_id = 11155111 # Sepolia oracle_address = "0x694AA1769357215DE4FAC081bf1f309aDC325306" # Oracle's decimals (Chainlink USD pairs are 8). decimals = "8" -# Trigger price in the oracle's native decimal units. Below this, sell. -trigger_price = "2500.00" +# Trigger price in the oracle's native decimal units. The Sepolia +# Chainlink ETH/USD feed reports a mocked value around $1681 at the +# time of the COW-1064 E2E run (2026-06-18). Setting the trigger +# *above* the live price + direction=below ensures the strategy fires +# on the first block. +trigger_price = "2000.00" # Order parameters. The owner pre-signs via GPv2Signing.setPreSignature # (on-chain, outside this module); the module submits the body with # Signature::PreSign on trigger. -owner = "0x70997970C51812dc3A010C7d01b50e0d17dc79C8" -sell_token = "0x6810e776880C02933D47DB1b9fc05908e5386b96" -buy_token = "0xfff9976782d46cc05630d1f6ebab18b2324d6b14" -sell_amount_wei = "1000000000000000000" -buy_amount_wei = "300000000000000000" +# +# E2E run pinning (COW-1064): test EOA on Sepolia with 0.05 ETH +# balance. Without a pre-sign + a WETH wrap the orderbook will reject +# with TransferSimulationFailed which the SDK classifies as +# TryNextBlock — that itself is a valid terminal marker (`backoff:` +# write to local-store) and proves the full submit path E2E. +owner = "0x7bF140727D27ea64b607E042f1225680B40ECa6A" +# WETH9 Sepolia (`wss://sepolia.etherscan.io/token/0xfff9976782d46cc05630d1f6ebab18b2324d6b14`). +sell_token = "0xfFf9976782d46CC05630D1f6eBAb18b2324d6B14" +# COW token Sepolia (verified on-chain: name="CoW Protocol Token", +# symbol="COW", decimals=18). +buy_token = "0x0625aFB445C3B6B7B929342a04A22599fd5dBB59" +# 0.005 WETH (small enough to fit in the 0.01 WETH wrap budget the +# E2E runbook recommends; large enough that the orderbook's min- +# quote endpoint actually returns a price). +sell_amount_wei = "5000000000000000" +# 20 COW (conservative; current quote on cow.fi/sepolia at the time +# of the E2E run is ~30 COW per 0.005 WETH so a 20 COW buy_amount +# leaves room for slippage without making the order too generous). +buy_amount_wei = "20000000000000000000" +# uint32::MAX = order never expires. valid_to_seconds = "4294967295" diff --git a/modules/examples/stop-loss/src/strategy.rs b/modules/examples/stop-loss/src/strategy.rs index 3ba79444..7128e255 100644 --- a/modules/examples/stop-loss/src/strategy.rs +++ b/modules/examples/stop-loss/src/strategy.rs @@ -390,6 +390,40 @@ mod tests { format!("{uid}") } + /// Regression test pinning the OrderUid produced by the COW-1064 + /// E2E run's `modules/examples/stop-loss/module.toml` config so an + /// operator can `setPreSignature(uid, true)` ahead of the run + /// without re-deriving the UID from the EIP-712 / domain- + /// separator dance. If this assertion ever flips, either: + /// (a) the module.toml has drifted from the pinned settings, or + /// (b) the EIP-712 type-hash / domain-separator changed, + /// and the runbook's `setPreSignature` step needs the new UID. + #[test] + fn cow_1064_e2e_settings_yield_expected_uid() { + let settings = Settings { + oracle_address: "0x694AA1769357215DE4FAC081bf1f309aDC325306" + .parse() + .unwrap(), + trigger_price_scaled: I256::try_from(200_000_000_000_i128).unwrap(), + owner: "0x7bF140727D27ea64b607E042f1225680B40ECa6A" + .parse() + .unwrap(), + sell_token: "0xfFf9976782d46CC05630D1f6eBAb18b2324d6B14" + .parse() + .unwrap(), + buy_token: "0x0625aFB445C3B6B7B929342a04A22599fd5dBB59" + .parse() + .unwrap(), + sell_amount: U256::from(5_000_000_000_000_000_u128), + buy_amount: U256::from(20_000_000_000_000_000_000_u128), + valid_to: u32::MAX, + }; + assert_eq!( + programmed_uid(&settings), + "0xc2b9cb4ea1ee5a86d8049ac09d8f494bf04cca0a68407285f31e2e6379800be87bf140727d27ea64b607e042f1225680b40eca6affffffff", + ); + } + #[test] fn idle_when_price_above_trigger() { let host = MockHost::new(); diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..5b2349be --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,134 @@ +# scripts/ — COW-1064 E2E automation + +Three-step automation for the COW-1064 E2E run on Sepolia. Wraps +the runbook (`docs/operations/e2e-testnet-runbook.md`) + the prep +punch list (`docs/operations/e2e-cow-1064-prep.md`) into shell +scripts so the operator only has to (a) fill in `.env` and +(b) decide when to stop. + +## One-time setup + +```bash +cp scripts/env-template scripts/.env +$EDITOR scripts/.env # fill in RPC URLs + EOA private key +``` + +`.env` is gitignored — secrets stay on disk, never enter chat, +never get committed. + +Required external tools: + +- `cargo` + the `wasm32-wasip2` target (already there if you've + built the workspace before). +- `cast` from foundry (`curl -L https://foundry.paradigm.xyz | bash && foundryup`). +- `jq`, `curl`, `python3` with `pip3 install eth-utils eth-abi pycryptodome`. + +## Running + +```bash +scripts/e2e-run.sh # boots engine, captures metrics baseline (~1 min) +scripts/e2e-onchain.sh # submits TWAP + EthFlow on-chain (~1 min, ~0.005 ETH) +# … engine runs for ~5 h to hit the 1500-block acceptance bar … +scripts/e2e-finish.sh # SIGINTs engine, captures end metrics, generates report +``` + +Three artefacts land in `docs/operations/e2e-reports/`: + +| File | Provenance | +|---|---| +| `engine-.log` | Full JSON-formatted supervisor log (~5 MB / 5 h). | +| `metrics-start-.txt` | `/metrics` snapshot at boot. | +| `metrics-end-.txt` | `/metrics` snapshot at SIGINT. | +| `e2e-report-.md` | Auto-filled COW-1064 report. Operator reviews + signs off + commits. | + +The first three are gitignored; the report is committed manually +once you've reviewed it. + +## Script details + +### `e2e-run.sh` + +- Renders `engine.e2e.toml` → `engine.e2e.local.toml` + (gitignored via `*.local.toml`) with `RPC_URL_SEPOLIA` + substituted in. Embedded URL key never reaches git. +- Cleans `data/e2e/` for a fresh local-store. +- Builds 5 modules + engine in `--release`. +- Launches via `nohup`; engine survives the parent shell exiting. +- Waits ≤ 60 s for `supervisor ready modules=5 chains=1`. +- Persists `ENGINE_PID`, `LOG_FILE`, `METRICS_START`, `START_TS`, + `START_ISO` into `scripts/.state` (gitignored). + +### `e2e-onchain.sh` + +Pre-flight: +- Derives the EOA address from `OPERATOR_PRIVATE_KEY` and asserts + it matches the pinned `0x7bF140727D27ea64b607E042f1225680B40ECa6A`. +- Asserts EOA balance ≥ 0.02 ETH. + +Required actions: +1. **TWAP** — `cast send ComposableCoW.create((handler,salt,staticInput),true)` + with the 516-byte pinned calldata. Fires + `ConditionalOrderCreated` → twap-monitor logs `watch:`. +2. **EthFlow** — calls `scripts/_ethflow_quote.py` to hit cow.fi + `/api/v1/quote`, encodes the returned `EthFlowOrder.Data`, + then `cast send EthFlow.createOrder` with the right msg.value. + Fires `OrderPlacement` → ethflow-watcher logs `submitted:`. + +Optional (gated on `RUN_OPTIONAL_PRESIGN=1` in `.env`): +3. `WETH9.deposit()` payable 0.01 ETH. +4. `GPv2Settlement.setPreSignature(uid, true)` with the pinned UID. +5. `WETH9.approve(GPv2VaultRelayer, 0.005 ETH)`. + +Each tx hash appended to `scripts/.state` so the report generator +can link them. + +> stop-loss already produces `submitted:{uid}` on the very first +> block (verified in run-prep smoke — the CoW orderbook accepts +> PreSign orders upfront). The optional path is only needed if you +> want the order to actually **settle** on-chain. + +### `e2e-finish.sh` + +- Captures `metrics-end-.txt`. +- Sends `SIGINT` to the engine PID. +- Waits ≤ 30 s for `graceful shutdown complete` in the log + (COW-1072 path). +- Escalates to `SIGKILL` if the engine is still alive after 30 s. +- Invokes `e2e-report-gen.sh` to write the filled-in report. + +### `e2e-report-gen.sh` + +Reads `LOG_FILE`, `METRICS_START`, `METRICS_END`, `START_ISO`, +`END_ISO`, and the `TX_*` hashes from `scripts/.state`; computes: + +- Chain coverage (first/last block from `block_number` log fields). +- Per-module first terminal marker timestamp + sample line. +- Delta of every `shepherd_*` Prometheus counter / histogram. +- ERROR + trapped + poisoned tallies. +- Per-row acceptance checklist (auto-checks block delta ≥ 1500, + marker per module, zero traps, zero poisons, zero ERRORs, + TWAP+EthFlow tx hashes present). + +Writes `e2e-report-.md` in `docs/operations/e2e-reports/`. +Operator: review + add anomalies (section 6) + sign off +(section 8) + commit with `git add -f`. + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `scripts/.env not found` | First run | `cp scripts/env-template scripts/.env && $EDITOR .env` | +| `cast wallet address failed` | bad PK format | Must be `0x` + 64 hex chars. No spaces. | +| `engine did not reach supervisor-ready in 60s` | RPC unreachable / config error | `tail -30 docs/operations/e2e-reports/engine-*.log` to see why | +| `cow.fi /quote returned 4xx` | Orderbook didn't like the quote params | Read the body in the error; usually a token-pair issue. Wait + retry if Sepolia orderbook is flaky. | +| `engine already running` | Prior run not finished | `scripts/e2e-finish.sh` (or `kill -INT $(grep ENGINE_PID scripts/.state | cut -d= -f2)`) | +| `block delta` in report is low | Run was too short | The acceptance bar is ≥ 1500 (~5 h). Anything less doesn't close COW-1064 even with all 5 markers. | + +## Re-running cleanly + +```bash +scripts/e2e-finish.sh # safe even if it's the only command — graceful exit +rm -rf data/e2e # wipe local-store +rm scripts/.state # wipe run state +scripts/e2e-run.sh # fresh start +``` diff --git a/scripts/_ethflow_quote.py b/scripts/_ethflow_quote.py new file mode 100755 index 00000000..beedbdcd --- /dev/null +++ b/scripts/_ethflow_quote.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""ethflow quote + tuple-encode helper. + +Called by scripts/e2e-onchain.sh. Hits the CoW Sepolia orderbook +`/api/v1/quote` endpoint for a native-ETH sell, then ABI-encodes the +EthFlowOrder.Data tuple the EthFlow contract expects as the +`createOrder` argument, plus the msg.value the operator must send. + +Output (stdout, two lines): + + CALLDATA=0x + VALUE_WEI= + +The script is deliberately fail-loud: any non-200 from cow.fi or a +quote shape we don't recognise aborts with a non-zero exit. +""" +from __future__ import annotations + +import json +import os +import sys +import urllib.error +import urllib.request + +from eth_abi import encode +from eth_utils import keccak + +COW_API = "https://api.cow.fi/sepolia/api/v1" +# CoW's quote endpoint rejects the native-ETH sentinel +# (`InvalidNativeSellToken`). EthFlow orders are quoted with the +# wrapped form (WETH9 Sepolia) as the sell side and the EthFlow +# contract handles the wrap on `createOrder` from `msg.value`. +WETH_SEPOLIA = "0xfFf9976782d46CC05630D1f6eBAb18b2324d6B14" +BUY_TOKEN = "0x0625aFB445C3B6B7B929342a04A22599fd5dBB59" # COW Sepolia + +EMPTY_APP_DATA_JSON = "{}" +EMPTY_APP_DATA_HASH = "0x" + keccak(EMPTY_APP_DATA_JSON.encode()).hex() + + +def fetch_quote(eoa: str, sell_amount_wei: int) -> dict: + body = { + "sellToken": WETH_SEPOLIA, + "buyToken": BUY_TOKEN, + "from": eoa, + "receiver": eoa, + "sellAmountBeforeFee": str(sell_amount_wei), + "kind": "sell", + "partiallyFillable": False, + "sellTokenBalance": "erc20", + "buyTokenBalance": "erc20", + "signingScheme": "eip1271", + "onchainOrder": True, + "appData": EMPTY_APP_DATA_JSON, + "appDataHash": EMPTY_APP_DATA_HASH, + } + req = urllib.request.Request( + f"{COW_API}/quote", + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Accept": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + return json.loads(resp.read()) + except urllib.error.HTTPError as e: + sys.exit(f"cow.fi /quote returned {e.code}: {e.read().decode(errors='replace')}") + + +def main() -> None: + eoa = sys.argv[1] + sell_amount_wei = int(sys.argv[2]) + + q = fetch_quote(eoa, sell_amount_wei) + inner = q["quote"] + quote_id = int(q["id"]) + fee_amount = int(inner["feeAmount"]) + buy_amount = int(inner["buyAmount"]) + valid_to = int(inner["validTo"]) + # The quote endpoint may have rebalanced sellAmount to reflect the + # fee; for an EthFlow order we honour the rebalanced value. + sell_amount = int(inner["sellAmount"]) + + # EthFlowOrder.Data: + # address buyToken; + # address receiver; + # uint256 sellAmount; + # uint256 buyAmount; + # bytes32 appData; + # uint256 feeAmount; + # uint32 validTo; + # bool partiallyFillable; + # int64 quoteId; + encoded = encode( + ["(address,address,uint256,uint256,bytes32,uint256,uint32,bool,int64)"], + [( + BUY_TOKEN, + eoa, + sell_amount, + buy_amount, + bytes.fromhex(EMPTY_APP_DATA_HASH[2:]), + fee_amount, + valid_to, + False, + quote_id, + )] + ) + selector = keccak(b"createOrder((address,address,uint256,uint256,bytes32,uint256,uint32,bool,int64))")[:4] + calldata = selector + encoded + value_wei = sell_amount + fee_amount + + print(f"CALLDATA=0x{calldata.hex()}") + print(f"VALUE_WEI={value_wei}") + print(f"# fee_amount={fee_amount} buy_amount={buy_amount} valid_to={valid_to} quote_id={quote_id} sell_amount={sell_amount}", + file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/e2e-finish.sh b/scripts/e2e-finish.sh new file mode 100755 index 00000000..160c3757 --- /dev/null +++ b/scripts/e2e-finish.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# scripts/e2e-finish.sh — gracefully wind down the COW-1064 E2E run. +# +# 1. Reads scripts/.state to find the engine PID + log file. +# 2. Captures /metrics → metrics-end-.txt before signalling. +# 3. Sends SIGINT to the engine. The graceful-shutdown path +# (COW-1072) writes `last_dispatched_block:{chain_id}` to the +# local-store + logs `graceful shutdown complete dispatched_ +# blocks=N dispatched_logs=M uptime_secs=K`. +# 4. Waits up to 30 s for that log line to appear. +# 5. Hands off to scripts/e2e-report-gen.sh which writes +# docs/operations/e2e-reports/e2e-report-YYYY-MM-DD.md. +# 6. Clears scripts/.state (run is closed). + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib.sh" + +require_cmd curl + +load_env + +[[ -f "$STATE_FILE" ]] || die "scripts/.state not found — was scripts/e2e-run.sh ever invoked?" +engine_pid="$(state_value ENGINE_PID)" || die "ENGINE_PID missing from .state" +log_file="$(state_value LOG_FILE)" || die "LOG_FILE missing from .state" +start_ts="$(state_value START_TS)" || die "START_TS missing from .state" + +ts="$(date -u +%Y%m%dT%H%M%SZ)" +metrics_end="$REPORTS_DIR/metrics-end-$ts.txt" +end_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +if ! kill -0 "$engine_pid" 2>/dev/null; then + warn "engine PID $engine_pid is not running anymore — skipping SIGINT, going straight to report" +else + log "capturing end-state metrics → $metrics_end" + if ! curl -sf http://127.0.0.1:9100/metrics > "$metrics_end"; then + warn "/metrics scrape failed before SIGINT — metrics-end will be empty" + : > "$metrics_end" + fi + + log "sending SIGINT to engine PID $engine_pid" + kill -INT "$engine_pid" + + log "waiting up to 30 s for graceful-shutdown log line" + shutdown_ok=0 + for _ in $(seq 1 30); do + if grep -q "graceful shutdown complete" "$log_file" 2>/dev/null; then + shutdown_ok=1 + break + fi + if ! kill -0 "$engine_pid" 2>/dev/null; then + break + fi + sleep 1 + done + if [[ $shutdown_ok -eq 0 ]]; then + warn "graceful-shutdown line never appeared; engine may have exited ungracefully" + fi + + # Final cleanup in case the process is still alive after 30s. + if kill -0 "$engine_pid" 2>/dev/null; then + warn "engine still alive after 30s — escalating to SIGKILL" + kill -KILL "$engine_pid" 2>/dev/null || true + fi +fi + +write_state "METRICS_END=$metrics_end" +write_state "END_TS=$ts" +write_state "END_ISO=$end_iso" + +log "generating report" +"$SCRIPT_DIR/e2e-report-gen.sh" + +log "report ready at $REPORTS_DIR/e2e-report-$(date -u +%Y-%m-%d).md" +log "run state file preserved at $STATE_FILE for reference (clear with: rm $STATE_FILE)" diff --git a/scripts/e2e-onchain.sh b/scripts/e2e-onchain.sh new file mode 100755 index 00000000..40c3d787 --- /dev/null +++ b/scripts/e2e-onchain.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# scripts/e2e-onchain.sh — execute the on-chain side of the COW-1064 +# E2E run. +# +# Pre-flight: +# - derive the EOA address from $OPERATOR_PRIVATE_KEY +# and assert it matches the pinned $TEST_EOA; +# - assert balance ≥ 0.02 ETH (covers 2 tx + slippage). +# +# Required actions (cover twap-monitor + ethflow-watcher markers): +# 1. ComposableCoW.create(...) — fires ConditionalOrderCreated; +# uses the 516-byte calldata pinned in lib.sh / +# e2e-cow-1064-prep.md so the TWAP order shape is reproducible. +# 2. EthFlow.createOrder(EthFlowOrder.Data) — fires OrderPlacement; +# tuple built dynamically from the cow.fi /quote response (the +# `quoteId` + `feeAmount` only exist after a quote, so this part +# is not pinned to a constant). +# +# Optional path (only if $RUN_OPTIONAL_PRESIGN=1 in scripts/.env): +# 3. WETH9.deposit() — wrap 0.01 ETH so stop-loss has a sell-side +# balance. +# 4. setPreSignature($EXPECTED_ORDER_UID, true) — enables the +# already-submitted stop-loss order for settlement. +# 5. WETH9.approve(GPv2VaultRelayer, 0.005 ETH) — sell-side +# allowance. +# +# Output: each tx hash is appended to scripts/.state under +# TX_=0x so the report generator can link them. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib.sh" + +require_cmd cast +require_cmd curl +require_cmd python3 + +python3 -c 'import eth_abi, eth_utils, eth_hash.auto' 2>/dev/null \ + || die "missing Python deps. Run: pip3 install eth-abi eth-utils \"eth-hash[pycryptodome]\"" + +load_env +[[ -n "${OPERATOR_PRIVATE_KEY:-}" ]] || die "OPERATOR_PRIVATE_KEY unset in scripts/.env" + +derived="$(cast wallet address --private-key "$OPERATOR_PRIVATE_KEY" 2>/dev/null)" \ + || die "cast wallet address failed — is OPERATOR_PRIVATE_KEY a valid 0x-prefixed 32-byte hex?" +# macOS still ships bash 3.2; ${var,,} (lowercase) is bash 4+ only, +# so we route through `tr` for case-insensitive comparison. +lower_derived="$(printf '%s' "$derived" | tr '[:upper:]' '[:lower:]')" +lower_expected="$(printf '%s' "$TEST_EOA" | tr '[:upper:]' '[:lower:]')" +if [[ "$lower_derived" != "$lower_expected" ]]; then + die "private key derives to $derived, expected $TEST_EOA — wrong EOA loaded" +fi +log "EOA: $derived" + +balance="$(cast balance "$TEST_EOA" --rpc-url "$RPC_URL_SEPOLIA_HTTP")" +log "EOA balance: $(python3 -c "print(f'{int(\"$balance\")/1e18:.6f} ETH')") ($balance wei)" +if (( balance < 20000000000000000 )); then # 0.02 ETH + die "EOA balance < 0.02 ETH — top up from a Sepolia faucet first" +fi + +# ── Action 1: ComposableCoW.create() ───────────────────────────────── + +twap_calldata="0x6bfae1ca000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000010000000000000000000000006cf1e9ca41f7611def408122793c358a3d11e5a5000000000000000000000000000000000000000000000000000000006670f00000000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000140000000000000000000000000fff9976782d46cc05630d1f6ebab18b2324d6b140000000000000000000000000625afb445c3b6b7b929342a04a22599fd5dbb5900000000000000000000000014995a1118caf95833e923faf8dd155721cd53c200000000000000000000000000000000000000000000000000038d7ea4c6800000000000000000000000000000000000000000000000000006f05b59d3b2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000025800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + +# Idempotency: if a prior invocation already wrote a TX_TWAP hash +# into .state, skip re-submitting (the ConditionalOrderCreated event +# already fired; re-running would either drop a tx with the same +# salt as a no-op, or — worse — bump the EOA's nonce for nothing). +if existing_twap="$(state_value TX_TWAP 2>/dev/null)" && [[ -n "${existing_twap:-}" ]]; then + log "TWAP already submitted in a prior invocation — skipping (tx: $existing_twap)" + tx_twap="$existing_twap" +else + log "submitting TWAP ComposableCoW.create() → $COMPOSABLE_COW" + tx_twap="$(cast send \ + --rpc-url "$RPC_URL_SEPOLIA_HTTP" \ + --private-key "$OPERATOR_PRIVATE_KEY" \ + --json \ + "$COMPOSABLE_COW" \ + "$twap_calldata" \ + | jq -r '.transactionHash')" + [[ "$tx_twap" =~ ^0x[a-fA-F0-9]{64}$ ]] || die "TWAP tx hash malformed: $tx_twap" + log " TWAP tx: $tx_twap" + log " Etherscan: https://sepolia.etherscan.io/tx/$tx_twap" + write_state "TX_TWAP=$tx_twap" +fi + +# ── Action 2: EthFlow.createOrder() ────────────────────────────────── + +if existing_ethflow="$(state_value TX_ETHFLOW 2>/dev/null)" && [[ -n "${existing_ethflow:-}" ]]; then + log "EthFlow already submitted in a prior invocation — skipping (tx: $existing_ethflow)" +else + log "fetching cow.fi /quote for EthFlow swap (0.005 ETH → COW)" + quote_out="$(python3 "$SCRIPT_DIR/_ethflow_quote.py" "$TEST_EOA" 5000000000000000)" \ + || die "EthFlow quote helper failed" + ethflow_calldata="$(echo "$quote_out" | grep '^CALLDATA=' | cut -d= -f2-)" + ethflow_value="$(echo "$quote_out" | grep '^VALUE_WEI=' | cut -d= -f2)" + [[ "$ethflow_calldata" =~ ^0x[a-fA-F0-9]+$ ]] || die "EthFlow calldata malformed" + [[ "$ethflow_value" =~ ^[0-9]+$ ]] || die "EthFlow value malformed: $ethflow_value" + log " msg.value = $ethflow_value wei ($(python3 -c "print(f'{$ethflow_value/1e18:.6f} ETH')"))" + + log "submitting EthFlow.createOrder() → $ETHFLOW" + tx_ethflow="$(cast send \ + --rpc-url "$RPC_URL_SEPOLIA_HTTP" \ + --private-key "$OPERATOR_PRIVATE_KEY" \ + --value "$ethflow_value" \ + --json \ + "$ETHFLOW" \ + "$ethflow_calldata" \ + | jq -r '.transactionHash')" + [[ "$tx_ethflow" =~ ^0x[a-fA-F0-9]{64}$ ]] || die "EthFlow tx hash malformed: $tx_ethflow" + log " EthFlow tx: $tx_ethflow" + log " Etherscan: https://sepolia.etherscan.io/tx/$tx_ethflow" + write_state "TX_ETHFLOW=$tx_ethflow" +fi + +# ── Optional actions ───────────────────────────────────────────────── + +if [[ "${RUN_OPTIONAL_PRESIGN:-0}" -eq 1 ]]; then + log "RUN_OPTIONAL_PRESIGN=1 → wrap WETH + setPreSignature + approve" + + log " WETH9.deposit() — wrapping 0.01 ETH" + tx_wrap="$(cast send --rpc-url "$RPC_URL_SEPOLIA_HTTP" --private-key "$OPERATOR_PRIVATE_KEY" --value 10000000000000000 --json "$WETH_SEPOLIA" "deposit()" | jq -r '.transactionHash')" + log " tx: $tx_wrap" + write_state "TX_WRAP=$tx_wrap" + + log " GPv2Settlement.setPreSignature($EXPECTED_ORDER_UID, true)" + tx_presign="$(cast send --rpc-url "$RPC_URL_SEPOLIA_HTTP" --private-key "$OPERATOR_PRIVATE_KEY" --json "$GPV2_SETTLEMENT" "setPreSignature(bytes,bool)" "$EXPECTED_ORDER_UID" true | jq -r '.transactionHash')" + log " tx: $tx_presign" + write_state "TX_PRESIGN=$tx_presign" + + log " WETH9.approve(GPv2VaultRelayer, 0.005 ETH)" + tx_approve="$(cast send --rpc-url "$RPC_URL_SEPOLIA_HTTP" --private-key "$OPERATOR_PRIVATE_KEY" --json "$WETH_SEPOLIA" "approve(address,uint256)" "$GPV2_VAULT_RELAYER" 5000000000000000 | jq -r '.transactionHash')" + log " tx: $tx_approve" + write_state "TX_APPROVE=$tx_approve" +else + log "RUN_OPTIONAL_PRESIGN=0 → skipping wrap/setPreSignature/approve" + log " (stop-loss still produces submitted:{uid} via the CoW orderbook" + log " pre-sign acceptance path; flip to 1 in .env to also enable on-chain settlement.)" +fi + +log "done. tail the engine log to watch markers land:" +log " tail -F $(state_value LOG_FILE)" diff --git a/scripts/e2e-report-gen.sh b/scripts/e2e-report-gen.sh new file mode 100755 index 00000000..2e2d169b --- /dev/null +++ b/scripts/e2e-report-gen.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env bash +# scripts/e2e-report-gen.sh — auto-fill the e2e-report template from +# the engine log + metrics-start/end snapshots + tx hashes captured +# during the run. +# +# Called by scripts/e2e-finish.sh, or stand-alone if the operator +# wants to regenerate the report after editing scripts/.state by +# hand. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib.sh" + +require_cmd jq +require_cmd python3 + +[[ -f "$STATE_FILE" ]] || die "scripts/.state not found" +log_file="$(state_value LOG_FILE)" || die "LOG_FILE missing" +metrics_start="$(state_value METRICS_START)" || die "METRICS_START missing" +metrics_end="$(state_value METRICS_END)" || die "METRICS_END missing" +start_iso="$(state_value START_ISO)" || start_iso="(unknown)" +end_iso="$(state_value END_ISO)" || end_iso="(unknown)" + +date_tag="$(date -u +%Y-%m-%d)" +report="$REPORTS_DIR/e2e-report-$date_tag.md" +template="$REPORTS_DIR/e2e-report.template.md" +[[ -f "$template" ]] || die "report template not found at $template" + +log "report → $report" +log "deriving chain-coverage + per-module markers from $log_file" + +python3 - "$log_file" "$metrics_start" "$metrics_end" "$start_iso" "$end_iso" "$template" "$report" "$STATE_FILE" <<'PY' +import json, os, re, sys +from pathlib import Path +from datetime import datetime, timezone + +LOG, M_START, M_END, START_ISO, END_ISO, TEMPLATE, OUT, STATE = sys.argv[1:9] + +# ── Parse engine log ───────────────────────────────────────────────── + +blocks = [] # list of dispatched block_numbers (per module, but we just want range) +markers = {m: [] for m in ("twap-monitor","ethflow-watcher","price-alert","balance-tracker","stop-loss")} +errors = [] +trapped = [] +poisoned = [] + +# Per-module terminal-state log fingerprints. Derived from the +# host.log() call sites inside modules/*/src/strategy.rs. Any one +# match against `message` (when the event carries that module's +# name) counts as a COW-1064 acceptance marker. +MARKER_PATTERNS = { + "twap-monitor": ["watch:", "indexed watch:", "poll watch:"], + "ethflow-watcher": ["ethflow submitted", "ethflow backoff", "ethflow dropped", "already submitted"], + "price-alert": ["TRIGGERED"], + # balance-tracker logs each per-block diff as + # "0x changed +N wei (prior=..., current=...)". + "balance-tracker": ["changed +", "changed -"], + "stop-loss": ["TRIGGERED", "retry on next block", "stop-loss submitted", + "stop-loss dropped", "already submitted", "submitted:"], +} + +def event_field(ev, key, default=None): + """tracing-subscriber's JSON formatter puts message / module / + block_number / target either at the top level (default) or + under a nested `fields` object (older versions / different + flatten modes). Look in both places.""" + if not isinstance(ev, dict): + return default + if key in ev: + return ev[key] + fields = ev.get("fields") + if isinstance(fields, dict) and key in fields: + return fields[key] + return default + +# Engine emits JSON to stdout by default (no --pretty-logs). Each +# line is one event. +with open(LOG) as f: + for line in f: + line = line.strip() + if not line: + continue + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + msg = event_field(ev, "message", "") or "" + module = event_field(ev, "module") + bn = event_field(ev, "block_number") + target = event_field(ev, "target", "") or "" + if bn is not None: + try: + blocks.append(int(bn)) + except (TypeError, ValueError): + pass + if isinstance(msg, str) and module in markers: + for needle in MARKER_PATTERNS.get(module, []): + if needle in msg: + markers[module].append({"ts": ev.get("timestamp",""), "level": ev.get("level",""), "msg": msg}) + break + if ev.get("level") == "ERROR" and target.startswith("nexum_engine"): + errors.append({"ts": ev.get("timestamp",""), "msg": msg}) + if "trapped" in msg and module: + trapped.append({"module": module, "msg": msg}) + if "poisoned" in msg and module: + poisoned.append({"module": module, "msg": msg}) + +first_block = min(blocks) if blocks else None +last_block = max(blocks) if blocks else None +block_delta = (last_block - first_block + 1) if blocks else 0 + +# ── Parse metrics ──────────────────────────────────────────────────── + +def parse_metrics(path): + """Return dict of {name+label_set: float}.""" + out = {} + if not os.path.isfile(path): + return out + with open(path) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + # name{labels} value OR name value + m = re.match(r"^(\w+)(?:\{([^}]*)\})?\s+(.+)$", line) + if not m: + continue + name, labels, val = m.groups() + try: + v = float(val) + except ValueError: + continue + key = name + "{" + (labels or "") + "}" + out[key] = v + return out + +ms = parse_metrics(M_START) +me = parse_metrics(M_END) + +def delta(name_prefix): + rows = [] + keys = sorted(set(k for k in {**ms, **me} if k.startswith(name_prefix))) + for k in keys: + s = ms.get(k, 0.0) + e = me.get(k, 0.0) + if e == 0.0 and s == 0.0: + continue + rows.append((k, s, e, e - s)) + return rows + +shepherd_keys = [ + "shepherd_event_latency_seconds", # histogram, will surface _sum/_count/_bucket + "shepherd_module_errors_total", + "shepherd_module_restarts_total", + "shepherd_module_poisoned", + "shepherd_chain_request_total", + "shepherd_cow_api_submit_total", + "shepherd_stream_reconnects_total", +] + +# ── Tx hashes (from .state) ────────────────────────────────────────── + +state_kv = {} +with open(STATE) as f: + for line in f: + line = line.strip() + if "=" in line: + k, v = line.split("=", 1) + state_kv[k] = v + +# ── Compose the report ─────────────────────────────────────────────── + +git_commit = os.popen("git -C $(dirname $0)/.. rev-parse HEAD 2>/dev/null").read().strip() or "(unknown)" +# Re-run cwd is /tmp/m3-base when invoked from finish.sh, but be safe: +git_commit = os.popen("git rev-parse HEAD 2>/dev/null").read().strip() or git_commit + +lines = [] +lines.append(f"# E2E testnet integration report — {datetime.now(timezone.utc).strftime('%Y-%m-%d')}") +lines.append("") +lines.append("> Auto-generated by `scripts/e2e-report-gen.sh`. Operator") +lines.append("> review each section + flesh out anomalies + sign off in") +lines.append("> section 8 before committing.") +lines.append("") +lines.append("## 1. Run metadata") +lines.append("") +lines.append("| Field | Value |") +lines.append("|---|---|") +lines.append("| Start (UTC) | " + START_ISO + " |") +lines.append("| End (UTC) | " + END_ISO + " |") +try: + sdt = datetime.fromisoformat(START_ISO.replace("Z","+00:00")) + edt = datetime.fromisoformat(END_ISO.replace("Z","+00:00")) + dur = edt - sdt + h, rem = divmod(int(dur.total_seconds()), 3600) + m, _ = divmod(rem, 60) + lines.append(f"| Wall clock | {h}h {m}m |") +except Exception: + lines.append("| Wall clock | (parse error) |") +lines.append(f"| Engine commit | `{git_commit}` |") +lines.append("| Engine config | `engine.e2e.local.toml` (rendered from `engine.e2e.toml`) |") +lines.append("| RPC provider | (filled by operator) |") +lines.append("") + +lines.append("## 2. Chain coverage") +lines.append("") +lines.append("| Chain | First block | Last block | Block delta |") +lines.append("|---|---|---|---|") +lines.append(f"| Sepolia (11155111) | {first_block if first_block is not None else 'n/a'} | {last_block if last_block is not None else 'n/a'} | {block_delta} |") +lines.append("") +bar = 1500 +lines.append(f"COW-1064 acceptance: block delta ≥ {bar} → " + ("**PASS**" if block_delta >= bar else "**FAIL**")) +lines.append("") + +lines.append("## 3. On-chain actions submitted") +lines.append("") +def tx_row(kind, label): + h = state_kv.get(f"TX_{kind}") + if not h: + return f"| {label} | _(not run)_ |" + return f"| {label} | [{h}](https://sepolia.etherscan.io/tx/{h}) |" +lines.append("| Action | Tx |") +lines.append("|---|---|") +lines.append(tx_row("TWAP", "TWAP ComposableCoW.create()")) +lines.append(tx_row("ETHFLOW", "EthFlow.createOrder()")) +lines.append(tx_row("WRAP", "WETH9.deposit() (optional)")) +lines.append(tx_row("PRESIGN", "setPreSignature (optional)")) +lines.append(tx_row("APPROVE", "WETH approve to GPv2VaultRelayer (optional)")) +lines.append("") + +lines.append("## 4. Per-module terminal-state markers") +lines.append("") +lines.append("| Module | First marker | Sample line |") +lines.append("|---|---|---|") +for m in ("twap-monitor","ethflow-watcher","price-alert","balance-tracker","stop-loss"): + if markers[m]: + first = markers[m][0] + # Truncate the marker line for the table + sample = first["msg"] + if len(sample) > 100: + sample = sample[:97] + "..." + sample = sample.replace("|", "\\|") + lines.append(f"| {m} | {first['ts']} | `{sample}` |") + else: + lines.append(f"| {m} | _(none observed)_ | |") +lines.append("") + +lines.append("## 5. Error counts (Prometheus delta)") +lines.append("") +lines.append("| Metric | Start | End | Delta |") +lines.append("|---|---|---|---|") +any_delta = False +for prefix in shepherd_keys: + for k, s, e, d in delta(prefix): + any_delta = True + lines.append(f"| `{k}` | {s:g} | {e:g} | {d:g} |") +if not any_delta: + lines.append("| _(no non-zero counters surfaced — check metrics files exist + endpoint was reachable)_ | | | |") +lines.append("") + +lines.append("## 6. Anomalies + defects") +lines.append("") +if errors: + lines.append(f"- `ERROR` lines from `nexum_engine::*`: **{len(errors)}** (first: `{errors[0]['msg'][:80]}`)") +if trapped: + lines.append(f"- `trapped` events: **{len(trapped)}** ({set(t['module'] for t in trapped)})") +if poisoned: + lines.append(f"- `poisoned` events: **{len(poisoned)}** ({set(p['module'] for p in poisoned)})") +if not (errors or trapped or poisoned): + lines.append("- _(no automatic anomalies surfaced. Operator: do a final spot-check of the engine log and add any human-noticed weirdness here, then file Linear issues for each.)_") +lines.append("") + +lines.append("## 7. Acceptance checklist (COW-1064)") +lines.append("") +def check(ok, label): + return f"- [{'x' if ok else ' '}] {label}" +lines.append(check(block_delta >= bar, f"block delta ≥ {bar} (got {block_delta})")) +five_markers = all(bool(markers[m]) for m in markers) +lines.append(check(five_markers, "all 5 modules emitted ≥ 1 terminal-state marker")) +zero_trap_modules = [] +for k, s, e, d in delta("shepherd_module_errors_total"): + if 'error_kind="trap"' in k and d > 0: + zero_trap_modules.append(k) +lines.append(check(not zero_trap_modules, f"shepherd_module_errors_total{{error_kind=\"trap\"}} == 0 (offenders: {zero_trap_modules or 'none'})")) +poisoned_keys = [k for k,v in me.items() if k.startswith("shepherd_module_poisoned") and v != 0.0] +lines.append(check(not poisoned_keys, f"no module poisoned at end (offenders: {poisoned_keys or 'none'})")) +lines.append(check(not errors, f"0 ERROR lines from nexum_engine::* (got {len(errors)})")) +lines.append(check(state_kv.get("TX_TWAP") and state_kv.get("TX_ETHFLOW"), "TWAP + EthFlow on-chain txs submitted")) +lines.append("") + +lines.append("## 8. Sign-off (operator)") +lines.append("") +lines.append("> Auto-generated report. Operator: in 1-2 sentences confirm whether this run is clean enough to unblock COW-1031 (7-day soak). If any acceptance row above is `[ ]`, file the defect in Linear before signing off.") +lines.append("") +lines.append("…") +lines.append("") + +lines.append("## 9. Attachments") +lines.append("") +lines.append(f"- Engine log: `{os.path.relpath(LOG, os.path.dirname(OUT))}`") +lines.append(f"- Metrics start: `{os.path.relpath(M_START, os.path.dirname(OUT))}`") +lines.append(f"- Metrics end: `{os.path.relpath(M_END, os.path.dirname(OUT))}`") +lines.append("") + +Path(OUT).write_text("\n".join(lines)) +print(f"wrote {OUT}", file=sys.stderr) +PY + +log "report written. Next: review + add anomalies + sign off + commit:" +log " \$EDITOR $report" +log " git add -f $report" +log " git commit -m 'ops(e2e): COW-1064 run report ${date_tag}'" diff --git a/scripts/e2e-run.sh b/scripts/e2e-run.sh new file mode 100755 index 00000000..e5f0fd6a --- /dev/null +++ b/scripts/e2e-run.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# scripts/e2e-run.sh — boot the COW-1064 E2E run. +# +# 1. Loads scripts/.env (RPC URLs, optional flags). +# 2. Renders engine.e2e.toml -> engine.e2e.local.toml with the +# operator's RPC URL (with key) substituted in. Local file is +# gitignored. +# 3. Cleans data/e2e for a fresh local-store. +# 4. Builds all 5 modules + the engine. +# 5. Launches nexum-engine via nohup, redirecting stdout/stderr to +# docs/operations/e2e-reports/engine-.log. JSON logs +# (no --pretty-logs) so e2e-report-gen.sh can mine them with jq. +# 6. Waits up to 60 s for the `supervisor ready modules=5 chains=1` +# line, exiting non-zero if it never appears. +# 7. Captures metrics-start.txt. +# 8. Persists engine PID, log path, and start-time to scripts/.state +# so e2e-onchain.sh + e2e-finish.sh can find them. +# 9. Prints the next-steps banner. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib.sh" + +require_cmd curl +require_cmd cargo +require_cmd python3 +require_cmd jq + +load_env + +if [[ -f "$STATE_FILE" ]]; then + if existing_pid="$(state_value ENGINE_PID || true)"; [[ -n "${existing_pid:-}" ]] && kill -0 "$existing_pid" 2>/dev/null; then + die "engine already running (PID $existing_pid). Run scripts/e2e-finish.sh first, or kill -INT $existing_pid manually." + fi + warn "stale state file $STATE_FILE — removing" + clear_state +fi + +mkdir -p "$REPORTS_DIR" + +render_engine_config + +log "cleaning local-store at $REPO_ROOT/data/e2e" +rm -rf "$REPO_ROOT/data/e2e" + +log "building 5 modules + engine (this can take a minute on first run)" +( + cd "$REPO_ROOT" + cargo build -p twap-monitor --target wasm32-wasip2 --release >/dev/null + cargo build -p ethflow-watcher --target wasm32-wasip2 --release >/dev/null + cargo build -p price-alert --target wasm32-wasip2 --release >/dev/null + cargo build -p balance-tracker --target wasm32-wasip2 --release >/dev/null + cargo build -p stop-loss --target wasm32-wasip2 --release >/dev/null + cargo build -p nexum-engine --release >/dev/null +) + +ts="$(date -u +%Y%m%dT%H%M%SZ)" +log_file="$REPORTS_DIR/engine-$ts.log" +metrics_start="$REPORTS_DIR/metrics-start-$ts.txt" +start_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + +log "launching engine — log: $log_file" +( + cd "$REPO_ROOT" + nohup "$REPO_ROOT/target/release/nexum-engine" \ + --engine-config "$REPO_ROOT/engine.e2e.local.toml" \ + >"$log_file" 2>&1 & + echo $! > "$STATE_FILE.pid.tmp" +) +engine_pid="$(cat "$STATE_FILE.pid.tmp")" +rm "$STATE_FILE.pid.tmp" + +log "waiting for supervisor-ready (PID $engine_pid)" +# The engine emits JSON to stdout (no --pretty-logs), so look for +# the message + modules + chains fields in the JSON shape rather +# than the pretty-printed `modules=5 chains=1` flat string. +ready=0 +for _ in $(seq 1 90); do + if grep -qE '"message":"supervisor ready"[^}]*"modules":5[^}]*"chains":1' "$log_file" 2>/dev/null \ + || grep -qE '"message":"supervisor ready"[^}]*"chains":1[^}]*"modules":5' "$log_file" 2>/dev/null; then + ready=1 + break + fi + if ! kill -0 "$engine_pid" 2>/dev/null; then + die "engine PID $engine_pid died before supervisor-ready. Tail: $(tail -20 "$log_file")" + fi + sleep 1 +done +[[ $ready -eq 1 ]] || die "engine did not reach supervisor-ready in 90s. Tail: $(tail -20 "$log_file")" + +log "capturing baseline metrics → $metrics_start" +curl -sf http://127.0.0.1:9100/metrics > "$metrics_start" \ + || die "/metrics scrape failed — is the metrics exporter bound?" + +{ + echo "ENGINE_PID=$engine_pid" + echo "LOG_FILE=$log_file" + echo "METRICS_START=$metrics_start" + echo "START_TS=$ts" + echo "START_ISO=$start_iso" +} > "$STATE_FILE" + +cat </dev/null + +EOF diff --git a/scripts/env-template b/scripts/env-template new file mode 100644 index 00000000..fd596eed --- /dev/null +++ b/scripts/env-template @@ -0,0 +1,48 @@ +# scripts/env-template — copy to scripts/.env and fill in. +# +# cp scripts/env-template scripts/.env +# $EDITOR scripts/.env +# +# scripts/.env is gitignored — secrets never leave your disk. +# The automation scripts source this file via `set -a; source .env; set +a`. + +# ── Required ────────────────────────────────────────────────────────── + +# Sepolia WS RPC URL. Engine subscribes to blocks + logs here. +# Public node throttles under sustained load; use a paid endpoint +# (Alchemy / Infura / drpc / QuickNode). Format: wss://… with key +# embedded. +RPC_URL_SEPOLIA="wss://YOUR_PROVIDER/sepolia/YOUR_KEY" + +# Sepolia HTTP RPC URL. Used by cast for on-chain submissions +# (the wss flavour rejects regular eth_sendRawTransaction). Same +# provider as RPC_URL_SEPOLIA, https:// scheme. +RPC_URL_SEPOLIA_HTTP="https://YOUR_PROVIDER/sepolia/YOUR_KEY" + +# Test EOA private key (0x-prefixed, 32 bytes hex). The EOA must: +# - hold ≥ 0.05 ETH on Sepolia (faucet); +# - match `owner` in modules/examples/stop-loss/module.toml; +# - be in the addresses list of modules/examples/balance-tracker/module.toml. +# +# Treat this file like any other credential: never commit, never +# share, never paste into chat. The scripts read this variable +# from disk only. +OPERATOR_PRIVATE_KEY="0xYOUR_PRIVATE_KEY" + +# ── Optional ────────────────────────────────────────────────────────── + +# How long the engine runs before scripts/e2e-finish.sh ends it +# (seconds). COW-1064 acceptance bar wants ≥ 1500 Sepolia blocks +# = ~5h at 12s blocks; default 21600 = 6h gives margin. +# +# For a smoke test of the automation itself, set this to 120 (2 min) +# — every script still runs end-to-end, but the chain-delta bar in +# the report won't clear acceptance. +RUN_DURATION_SECONDS=21600 + +# Set to 1 to also run the optional setPreSignature + WETH wrap + +# approve sequence for stop-loss. Without this, stop-loss still +# produces a `submitted:{uid}` terminal marker (the orderbook +# accepts PreSign orders upfront — verified in the run-prep smoke); +# with this, the submitted order is also settleable on-chain. +RUN_OPTIONAL_PRESIGN=0 diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100644 index 00000000..74026c9d --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,86 @@ +# scripts/lib.sh — shared bash helpers for the COW-1064 E2E automation. +# Source this from each e2e-*.sh; do not run it directly. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ENV_FILE="$SCRIPT_DIR/.env" +STATE_FILE="$SCRIPT_DIR/.state" +REPORTS_DIR="$REPO_ROOT/docs/operations/e2e-reports" + +# Pinned identities — match docs/operations/e2e-cow-1064-prep.md +# section 0. If you change one, change them in lock-step and re-run +# `cargo test -p stop-loss --lib cow_1064`. +TEST_EOA="0x7bF140727D27ea64b607E042f1225680B40ECa6A" +TEST_SAFE="0x14995a1118Caf95833e923faf8Dd155721cd53c2" +COMPOSABLE_COW="0xfdaFc9d1902f4e0b84f65F49f244b32b31013b74" +TWAP_HANDLER="0x6cF1e9cA41f7611dEf408122793c358a3d11E5a5" +ETHFLOW="0xbA3cB449bD2B4ADddBc894D8697F5170800EAdeC" +GPV2_SETTLEMENT="0x9008D19f58AAbD9eD0D60971565AA8510560ab41" +GPV2_VAULT_RELAYER="0xc92e8bdf79f0507f65a392b0ab4667716bfe0110" +WETH_SEPOLIA="0xfFf9976782d46CC05630D1f6eBAb18b2324d6B14" +COW_SEPOLIA="0x0625aFB445C3B6B7B929342a04A22599fd5dBB59" +EXPECTED_ORDER_UID="0xc2b9cb4ea1ee5a86d8049ac09d8f494bf04cca0a68407285f31e2e6379800be87bf140727d27ea64b607e042f1225680b40eca6affffffff" + +log() { printf "\033[1;34m[e2e]\033[0m %s\n" "$*" >&2; } +warn() { printf "\033[1;33m[e2e WARN]\033[0m %s\n" "$*" >&2; } +die() { printf "\033[1;31m[e2e FAIL]\033[0m %s\n" "$*" >&2; exit 1; } + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1 — install before running" +} + +load_env() { + [[ -f "$ENV_FILE" ]] || die "scripts/.env not found. Run: cp scripts/env-template scripts/.env && \$EDITOR scripts/.env" + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a + [[ -n "${RPC_URL_SEPOLIA:-}" ]] || die "RPC_URL_SEPOLIA unset in scripts/.env" + [[ -n "${RPC_URL_SEPOLIA_HTTP:-}" ]] || die "RPC_URL_SEPOLIA_HTTP unset in scripts/.env" + [[ "${RPC_URL_SEPOLIA}" == wss* ]] || die "RPC_URL_SEPOLIA must be wss:// (engine uses eth_subscribe)" + [[ "${RPC_URL_SEPOLIA_HTTP}" == http* ]] || die "RPC_URL_SEPOLIA_HTTP must be http(s)://" +} + +# Render engine.e2e.toml -> engine.e2e.local.toml with the rpc_url +# substituted in. engine.e2e.local.toml is gitignored (via *.local.toml) +# so the URL with embedded key never leaks into git history. +render_engine_config() { + local src="$REPO_ROOT/engine.e2e.toml" + local dst="$REPO_ROOT/engine.e2e.local.toml" + [[ -f "$src" ]] || die "engine.e2e.toml not found at $src" + + # We do the substitution via python -c to avoid any sed escape + # issues with the URL. + RPC_URL_SEPOLIA="$RPC_URL_SEPOLIA" python3 - "$src" "$dst" <<'PY' +import os, re, sys +src, dst = sys.argv[1], sys.argv[2] +rpc = os.environ["RPC_URL_SEPOLIA"] +with open(src) as f: + content = f.read() +# Match the rpc_url line inside [chains.11155111] block. The toml is +# small + we control its shape — a regex is safe here. +new = re.sub( + r'(\[chains\.11155111\]\nrpc_url\s*=\s*)"[^"]*"', + lambda m: m.group(1) + f'"{rpc}"', + content, + count=1, +) +if new == content: + sys.exit("could not substitute rpc_url in engine.e2e.toml") +with open(dst, "w") as f: + f.write(new) +PY + log "rendered $dst" +} + +write_state() { printf '%s\n' "$@" >> "$STATE_FILE"; } +read_state() { [[ -f "$STATE_FILE" ]] && cat "$STATE_FILE" || true; } +clear_state() { rm -f "$STATE_FILE"; } + +state_value() { + local key="$1" + [[ -f "$STATE_FILE" ]] || return 1 + grep -E "^${key}=" "$STATE_FILE" | tail -1 | cut -d= -f2- +}