diff --git a/apps/bot/README.md b/apps/bot/README.md index a88cff2..7e0bae1 100644 --- a/apps/bot/README.md +++ b/apps/bot/README.md @@ -68,11 +68,11 @@ Stable event types: - `bot.transaction.failed` - `bot.iteration.failed` -No-action iterations emit `bot.decision.skipped` with `reason` and evidence. Build-time skip reasons `no_actions` and `match_value_not_above_fee` include a `decision` transcript. The pre-build safety skip `capital_below_minimum` exits with code `2` and includes zero `actions` plus `state` evidence instead of a `decision` transcript because match, rebalance, fee, and transaction shape were not evaluated. Rebalance no-op decisions include policy-owned reasons such as `insufficient_output_slots`, `low_ickb_ckb_reserve_unavailable`, `target_ickb_not_exceeded`, and `no_ready_withdrawal_selection`. +No-action iterations emit `bot.decision.skipped` with `reason` and evidence. Build-time skip reasons `no_actions` and `match_value_not_above_fee` include a `decision` transcript. The pre-build safety skip `capital_below_minimum` exits with code `2` and includes zero `actions` plus `state` evidence instead of a `decision` transcript because match, rebalance, fee, and transaction shape were not evaluated. `bot.iteration.failed` includes a redacted `error` summary plus `retryable` and `terminal` booleans from the bot retry policy. Rebalance no-op decisions include policy-owned reasons such as `insufficient_output_slots`, `low_ickb_ckb_reserve_unavailable`, `target_ickb_not_exceeded`, and `no_ready_withdrawal_selection`. The decision transcript groups evidence under `chainTip`, `balances`, `orders`, `withdrawals`, `poolDeposits`, `match`, `rebalance`, `actions`, `fee`, `transactionShape`, `exchangeRatio`, and `depositCapacity`. Transaction events summarize action counts, fee, fee rate, tx hash, confirmation status, check count, elapsed time, and transaction shape counts. -JSON `"maxIterations":1` makes `pnpm --filter ./apps/bot start` exit with code `0` after one terminal iteration when that terminal outcome is a skipped decision, committed transaction, non-safety transaction failure, or non-safety iteration failure. Safety stops still keep their nonzero behavior: low capital and confirmation timeouts after broadcast exit with code `2`. Omitting `maxIterations` keeps the default infinite loop. +JSON `"maxIterations":1` makes `pnpm --filter ./apps/bot start` exit with code `0` after one terminal iteration when that terminal outcome is a skipped decision, committed transaction, non-safety transaction failure, or non-safety iteration failure. Retryable iteration failures are logged with `terminal:false` and do not count toward `maxIterations`. Safety stops still keep their nonzero behavior: low capital and confirmation timeouts after broadcast exit with code `2`. Omitting `maxIterations` keeps the default infinite loop. Structured events should contain public evidence and summaries needed to understand bot behavior. Do not add private keys, seed phrases, mnemonics, or other secrets to log payloads; omit noisy public fields at the call site instead of relying on redaction. diff --git a/apps/bot/src/index.test.ts b/apps/bot/src/index.test.ts index 071fd90..7bf51d1 100644 --- a/apps/bot/src/index.test.ts +++ b/apps/bot/src/index.test.ts @@ -9,7 +9,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { TARGET_ICKB_BALANCE } from "./policy.js"; -import { completeTerminalIteration, readBotRuntimeConfig } from "./index.js"; +import { + completeTerminalIteration, + isRetryableBotError, + iterationFailureEventFields, + readBotRuntimeConfig, +} from "./index.js"; import { buildTransaction, collectPoolDeposits, postTransactionPlainCkbBalance } from "./runtime.js"; afterEach(() => { @@ -163,6 +168,29 @@ describe("completeTerminalIteration", () => { }); }); +describe("bot retryable iteration failures", () => { + it("treats chain-tip races and fetch transport failures as retryable", () => { + expect(isRetryableBotError(new Error("L1 state scan crossed chain tip; retry with a fresh state"))).toBe(true); + expect(isRetryableBotError(new TypeError("fetch failed"))).toBe(true); + expect(isRetryableBotError(new Error("fetch failed"))).toBe(false); + expect(isRetryableBotError(new Error("deterministic build failure"))).toBe(false); + }); + + it("emits retryability metadata from the same retry decision", () => { + expect(iterationFailureEventFields(new TypeError("fetch failed"))).toMatchObject({ + retryable: true, + terminal: false, + error: { name: "TypeError", message: "fetch failed" }, + }); + + expect(iterationFailureEventFields(new Error("deterministic build failure"))).toMatchObject({ + retryable: false, + terminal: true, + error: { name: "Error", message: "deterministic build failure" }, + }); + }); +}); + describe("readBotRuntimeConfig", () => { it("requires a JSON config file", async () => { await expect(readBotRuntimeConfig({})).rejects.toThrow("Empty env BOT_CONFIG_FILE"); @@ -355,6 +383,30 @@ describe("buildTransaction", () => { ckbDelta: 0n, udtDelta: 0n, partials: [], + diagnostics: { + orderCount: 1, + allowance: { ckbValue: 0n, udtValue: 0n }, + ckbAllowanceStep: 1n, + udtAllowanceStep: 1n, + ckbMiningFee: 1n, + directions: { + ckbToUdt: { matchableCount: 0 }, + udtToCkb: { matchableCount: 1, minAllowance: 1n, maxMatch: 2n }, + }, + candidates: { + total: 2, + viable: 1, + positiveGain: 0, + rejected: { + maxPartials: 0, + duplicateOrder: 0, + insufficientCkbAllowance: 1, + insufficientUdtAllowance: 0, + nonPositiveGain: 1, + }, + bestGain: 0n, + }, + }, }); const completeTransaction = vi.fn(); @@ -369,6 +421,20 @@ describe("buildTransaction", () => { reason: "no_actions", decision: { rebalance: { kind: "none", reason: "target_ickb_not_exceeded" }, + match: { + diagnostics: { + orderCount: 1, + directions: { + udtToCkb: { matchableCount: 1, minAllowance: 1n, maxMatch: 2n }, + }, + candidates: { + rejected: { + insufficientCkbAllowance: 1, + nonPositiveGain: 1, + }, + }, + }, + }, actions: { collectedOrders: 0, completedDeposits: 0, diff --git a/apps/bot/src/index.ts b/apps/bot/src/index.ts index 79dd48e..c63ad54 100644 --- a/apps/bot/src/index.ts +++ b/apps/bot/src/index.ts @@ -19,6 +19,7 @@ import { sleep, STOP_EXIT_CODE, type RuntimeConfig, + type SecretRedactionContext, verifyChainPreflight, } from "@ickb/node-utils"; import { @@ -149,10 +150,9 @@ async function main(): Promise { } } catch (error) { stopAfterLog = handleLoopError(executionLog, error, secrets); - events.emit(iterationId, "bot.iteration.failed", { - error: errorSummary(error, secrets), - }); - if (isRetryableBotError(error)) { + const failure = iterationFailureEventFields(error, secrets); + events.emit(iterationId, "bot.iteration.failed", failure); + if (failure.retryable) { logExecution(executionLog, startTime); continue; } @@ -187,6 +187,19 @@ export function completeTerminalIteration( }; } +export function iterationFailureEventFields(error: unknown, secrets: SecretRedactionContext = {}): { + error: Record | string; + retryable: boolean; + terminal: boolean; +} { + const retryable = isRetryableBotError(error); + return { + error: errorSummary(error, secrets), + retryable, + terminal: !retryable, + }; +} + async function readBotState(runtime: Runtime): Promise { const accountLocks = await signerAccountLocks(runtime.signer, runtime.primaryLock); const { system, user, account } = await runtime.sdk.getL1AccountState( @@ -243,8 +256,11 @@ function outPointKey(outPoint: ccc.OutPoint): string { const fmtCkb = formatCkb; -function isRetryableBotError(error: unknown): boolean { - return error instanceof Error && error.message.includes("L1 state scan crossed chain tip"); +export function isRetryableBotError(error: unknown): boolean { + return error instanceof Error && ( + error.message.includes("L1 state scan crossed chain tip") || + (error instanceof TypeError && error.message === "fetch failed") + ); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/apps/bot/src/runtime.ts b/apps/bot/src/runtime.ts index a449769..5dcf96d 100644 --- a/apps/bot/src/runtime.ts +++ b/apps/bot/src/runtime.ts @@ -7,6 +7,8 @@ import { } from "@ickb/core"; import { OrderManager, + type Match, + type MatchDiagnostics, type OrderCell, type OrderGroup, } from "@ickb/order"; @@ -126,6 +128,7 @@ export interface BotDecisionTranscript { ckbDelta: bigint; udtDelta: bigint; value?: bigint; + diagnostics?: MatchDiagnostics; }; rebalance: { kind: RebalancePlan["kind"]; @@ -397,7 +400,7 @@ function buildDecisionTranscript({ tx, }: { state: BotState; - match: { partials: readonly unknown[]; ckbDelta: bigint; udtDelta: bigint }; + match: Pick; rebalance: RebalancePlan; outputSlots: number; actions: BotActions; @@ -410,6 +413,7 @@ function buildDecisionTranscript({ partialCount: match.partials.length, ckbDelta: match.ckbDelta, udtDelta: match.udtDelta, + ...(match.diagnostics === undefined ? {} : { diagnostics: match.diagnostics }), }, rebalance: rebalanceSummary(rebalance, outputSlots, state, match), actions, diff --git a/apps/supervisor/src/index.test.ts b/apps/supervisor/src/index.test.ts index a24ada0..011bcbd 100644 --- a/apps/supervisor/src/index.test.ts +++ b/apps/supervisor/src/index.test.ts @@ -961,6 +961,17 @@ describe("classification", () => { }), botEvent("bot.transaction.built", { actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 1, withdrawalRequests: 0, withdrawals: 0 }, + decision: { + match: { + diagnostics: { + directions: { + ckbToUdt: { matchableCount: 5 }, + udtToCkb: { matchableCount: 6 }, + }, + candidates: { viable: 7, positiveGain: 8 }, + }, + }, + }, }), botEvent("bot.transaction.committed", { txHash: txHash("33"), status: "committed" }), ].map(JSON.stringify).join("\n"); @@ -972,12 +983,66 @@ describe("classification", () => { marketOrderCount: 4, userOrderCount: 0, receiptCount: 1, + ckbToUdtMatchableOrderCount: 5, + udtToCkbMatchableOrderCount: 6, + viableMatchCandidateCount: 7, + positiveGainMatchCandidateCount: 8, readyPoolDepositCount: 2, nearReadyPoolDepositCount: 1, futurePoolDepositCount: 3, }); }); + it("keeps match diagnostics tied to the matching state-read iteration", () => { + const stdout = [ + botEvent("bot.state.read", { + iterationId: 1, + orders: { marketCount: 4, userCount: 0, receiptCount: 1 }, + poolDeposits: { readyCount: 2, nearReadyCount: 1, futureCount: 3 }, + }), + botEvent("bot.transaction.built", { + iterationId: 1, + actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 1, withdrawalRequests: 0, withdrawals: 0 }, + decision: { + match: { + diagnostics: { + directions: { + ckbToUdt: { matchableCount: 5 }, + udtToCkb: { matchableCount: 6 }, + }, + candidates: { viable: 7, positiveGain: 8 }, + }, + }, + }, + }), + botEvent("bot.state.read", { + iterationId: 2, + orders: { marketCount: 9, userCount: 1, receiptCount: 0 }, + poolDeposits: { readyCount: 0, nearReadyCount: 0, futureCount: 0 }, + }), + botEvent("bot.iteration.failed", { + iterationId: 2, + retryable: true, + terminal: false, + error: { name: "TypeError", message: "fetch failed" }, + }), + ].map(JSON.stringify).join("\n"); + const classification = classifyActorResult("bot", commandResult("bot", stdout)); + + expect(classification.publicState).toEqual({ + marketOrderCount: 9, + userOrderCount: 1, + receiptCount: 0, + ckbToUdtMatchableOrderCount: undefined, + udtToCkbMatchableOrderCount: undefined, + viableMatchCandidateCount: undefined, + positiveGainMatchCandidateCount: undefined, + readyPoolDepositCount: 0, + nearReadyPoolDepositCount: 0, + futurePoolDepositCount: 0, + }); + }); + it("rejects committed bot evidence without a valid tx hash", () => { const stdout = [ botEvent("bot.transaction.built", { @@ -1152,6 +1217,11 @@ describe("classification", () => { expect(safeArtifactText(JSON.stringify({ witnesses: ["0xsignature"], inputs: [] }))).toBe( "\n", ); + expect(safeArtifactText(JSON.stringify({ + transactionShape: { inputs: 1, outputs: 2, cellDeps: 3, headerDeps: 4, witnesses: 5 }, + }))).toBe( + JSON.stringify({ transactionShape: { inputs: 1, outputs: 2, cellDeps: 3, headerDeps: 4, witnesses: 5 } }), + ); expect(safeArtifactText(JSON.stringify({ system: { tip: { hash: txHash("aa") } } }))).toBe( JSON.stringify({ system: { tip: { hash: txHash("aa") } } }), ); diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 778b1ea..3a95795 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -150,6 +150,10 @@ export interface PublicStateAssumption { marketOrderCount?: number; userOrderCount?: number; receiptCount?: number; + ckbToUdtMatchableOrderCount?: number; + udtToCkbMatchableOrderCount?: number; + viableMatchCandidateCount?: number; + positiveGainMatchCandidateCount?: number; readyPoolDepositCount?: number; nearReadyPoolDepositCount?: number; futurePoolDepositCount?: number; @@ -1693,20 +1697,38 @@ function latestBotActions(records: Record[]): ActionCounts | un } function latestPublicState(records: Record[]): PublicStateAssumption | undefined { + const matchDiagnosticsByIteration = new Map>(); for (let index = records.length - 1; index >= 0; index -= 1) { const record = records[index]; if (record === undefined) { continue; } - if (stringField(record, "type") !== "bot.state.read") { + const type = stringField(record, "type"); + const iterationId = numberField(record, "iterationId"); + if (iterationId !== undefined && (type === "bot.decision.skipped" || type === "bot.transaction.built")) { + const matchDiagnostics = optionalRecordField(optionalRecordField(recordField(record, "decision"), "match"), "diagnostics"); + if (matchDiagnostics !== undefined && !matchDiagnosticsByIteration.has(iterationId)) { + matchDiagnosticsByIteration.set(iterationId, matchDiagnostics); + } + } + if (type !== "bot.state.read") { continue; } const orders = recordField(record, "orders"); const poolDeposits = recordField(record, "poolDeposits"); + const latestMatchDiagnostics = iterationId === undefined ? undefined : matchDiagnosticsByIteration.get(iterationId); + const directions = optionalRecordField(latestMatchDiagnostics, "directions"); + const ckbToUdt = optionalRecordField(directions, "ckbToUdt"); + const udtToCkb = optionalRecordField(directions, "udtToCkb"); + const candidates = optionalRecordField(latestMatchDiagnostics, "candidates"); return { marketOrderCount: numberField(orders, "marketCount"), userOrderCount: numberField(orders, "userCount"), receiptCount: numberField(orders, "receiptCount"), + ckbToUdtMatchableOrderCount: numberField(ckbToUdt, "matchableCount"), + udtToCkbMatchableOrderCount: numberField(udtToCkb, "matchableCount"), + viableMatchCandidateCount: numberField(candidates, "viable"), + positiveGainMatchCandidateCount: numberField(candidates, "positiveGain"), readyPoolDepositCount: numberField(poolDeposits, "readyCount"), nearReadyPoolDepositCount: numberField(poolDeposits, "nearReadyCount"), futurePoolDepositCount: numberField(poolDeposits, "futureCount"), @@ -1765,7 +1787,7 @@ function containsSecretLeak(text: string): boolean { } function containsTransactionLeak(text: string): boolean { - return /["']?(witnesses|cellDeps|headerDeps|inputs|outputs|outputsData)["']?\s*:/iu.test(text); + return /["']?(witnesses|cellDeps|headerDeps|inputs|outputs|outputsData)["']?\s*:\s*\[/iu.test(text); } export function safeArtifactText(text: string): string { @@ -1826,6 +1848,10 @@ function recordField(record: Record, key: string): Record | undefined, key: string): Record | undefined { + return record === undefined ? undefined : recordField(record, key); +} + function stringField(record: Record | undefined, key: string): string | undefined { const value = record?.[key]; return typeof value === "string" ? value : undefined; diff --git a/apps/tester/src/freshMatchableOrderSkip.ts b/apps/tester/src/freshMatchableOrderSkip.ts index 7fde3fd..348a104 100644 --- a/apps/tester/src/freshMatchableOrderSkip.ts +++ b/apps/tester/src/freshMatchableOrderSkip.ts @@ -1,8 +1,9 @@ import { ccc } from "@ckb-ccc/core"; -import { type OrderGroup } from "@ickb/order"; +import { ickbExchangeRatio } from "@ickb/core"; +import { OrderManager, type OrderGroup } from "@ickb/order"; import { type Runtime } from "./runtime.js"; -const MAX_ELAPSED_BLOCKS = 100800n; +const MAX_ELAPSED_BLOCKS = 5400n; type FreshMatchableOrderSkip = | { @@ -21,11 +22,12 @@ export async function freshMatchableOrderSkip( runtime: Runtime, orders: OrderGroup[], tip: ccc.ClientBlockHeader, + feeRate: ccc.Num, ): Promise { const tx2BlockNumber = new Map(); for (const group of orders) { - if (!group.order.isMatchable()) { + if (!isActionableOrder(group, tip, feeRate)) { continue; } @@ -52,3 +54,13 @@ export async function freshMatchableOrderSkip( } } } + +function isActionableOrder(group: OrderGroup, tip: ccc.ClientBlockHeader, feeRate: ccc.Num): boolean { + const { order } = group; + return OrderManager.bestMatch( + [order], + { ckbValue: ccc.fixedPointFrom(1000000), udtValue: ccc.fixedPointFrom(1000000) }, + ickbExchangeRatio(tip), + { feeRate, ckbAllowanceStep: ccc.fixedPointFrom(1), maxPartials: 1 }, + ).partials.length > 0; +} diff --git a/apps/tester/src/index.test.ts b/apps/tester/src/index.test.ts index 8870e8e..ca5b164 100644 --- a/apps/tester/src/index.test.ts +++ b/apps/tester/src/index.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { ccc } from "@ckb-ccc/core"; +import { OrderCell, OrderData, Ratio } from "@ickb/order"; import { byte32FromByte, headerLike, script } from "@ickb/testkit"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -7,6 +8,7 @@ import { join } from "node:path"; import { freshMatchableOrderSkip } from "./freshMatchableOrderSkip.js"; import { isRetryableTesterError, + isUnrepresentableTesterEstimateError, planTesterTransaction, postTransactionPlainCkbBalance, randomTesterScenario, @@ -102,6 +104,13 @@ describe("isRetryableTesterError", () => { }); }); +describe("isUnrepresentableTesterEstimateError", () => { + it("recognizes fee-adjusted ratio overflow as an unbuildable tester estimate", () => { + expect(isUnrepresentableTesterEstimateError(new Error("Ratio scale exceeds Uint64"))).toBe(true); + expect(isUnrepresentableTesterEstimateError(new Error("L1 state scan crossed chain tip; retry with a fresh state"))).toBe(false); + }); +}); + describe("planTesterTransaction", () => { it("does not silently downsize extra-large limit orders", () => { const depositCapacity = 1000n; @@ -133,8 +142,8 @@ describe("planTesterTransaction", () => { expect(planTesterTransaction(state, 1000n, "all-ckb-limit-order")).toEqual({ direction: "ckb-to-ickb", - amount: ccc.fixedPointFrom(647500), - ckbAmount: ccc.fixedPointFrom(647500), + amount: ccc.fixedPointFrom(647000), + ckbAmount: ccc.fixedPointFrom(647000), udtAmount: 0n, orderCount: 1, }); @@ -145,8 +154,8 @@ describe("planTesterTransaction", () => { expect(planTesterTransaction(state, 1000n, "two-ckb-to-ickb-limit-orders")).toEqual({ direction: "ckb-to-ickb", - amount: ccc.fixedPointFrom(647500), - ckbAmount: ccc.fixedPointFrom(647500), + amount: ccc.fixedPointFrom(647000), + ckbAmount: ccc.fixedPointFrom(647000), udtAmount: 0n, orderCount: 2, }); @@ -217,8 +226,8 @@ describe("planTesterTransaction", () => { expect(planTesterTransaction(state, 1000n, "mixed-direction-limit-orders")).toEqual({ direction: "ckb-to-ickb", - amount: ccc.fixedPointFrom(647623), - ckbAmount: ccc.fixedPointFrom(647500), + amount: ccc.fixedPointFrom(647123), + ckbAmount: ccc.fixedPointFrom(647000), udtAmount: ccc.fixedPointFrom(123), orderCount: 2, }); @@ -282,7 +291,7 @@ describe("planTesterTransaction", () => { availableIckbBalance: ccc.fixedPointFrom(123), }), "multi-order-limit-orders")).toBe("two-ickb-to-ckb-limit-orders"); expect(resolveTesterScenario(testerState({ - availableCkbBalance: ccc.fixedPointFrom(2500) + 1n, + availableCkbBalance: ccc.fixedPointFrom(3000) + 1n, availableIckbBalance: ccc.fixedPointFrom(123), }), "multi-order-limit-orders")).toBe("mixed-direction-limit-orders"); expect(() => resolveTesterScenario(testerState({ @@ -343,7 +352,8 @@ describe("freshMatchableOrderSkip", () => { await expect(freshMatchableOrderSkip( runtime as never, [matchableOrder(txHash)], - headerLike({ number: 200000n }), + headerLike({ number: 200000n, epoch: ccc.Epoch.from([0n, 0n, 1n]) }), + 0n, )).resolves.toEqual({ reason: "matchable-order-transaction-missing", txHash, @@ -361,13 +371,14 @@ describe("freshMatchableOrderSkip", () => { await expect(freshMatchableOrderSkip( runtime as never, [matchableOrder(txHash)], - headerLike({ number: 200000n }), + headerLike({ number: 105400n, epoch: ccc.Epoch.from([0n, 0n, 1n]) }), + 0n, )).resolves.toEqual({ reason: "fresh-matchable-order", txHash, blockNumber: 100000n, - tipNumber: 200000n, - maxElapsedBlocks: 100800n, + tipNumber: 105400n, + maxElapsedBlocks: 5400n, }); }); @@ -381,9 +392,41 @@ describe("freshMatchableOrderSkip", () => { await expect(freshMatchableOrderSkip( runtime as never, [matchableOrder(byte32FromByte("33")), nonMatchableOrder(byte32FromByte("44"))], - headerLike({ number: 200801n }), + headerLike({ number: 105401n, epoch: ccc.Epoch.from([0n, 0n, 1n]) }), + 0n, )).resolves.toBeUndefined(); }); + + it("does not skip fresh owned orders that are not marketable at the midpoint", async () => { + const runtime = { + client: { + getTransaction: (): Promise<{ blockNumber: bigint }> => Promise.resolve({ blockNumber: 100000n }), + }, + }; + + await expect(freshMatchableOrderSkip( + runtime as never, + [unmarketableOrder(byte32FromByte("45"))], + headerLike({ number: 105400n, epoch: ccc.Epoch.from([0n, 0n, 1n]) }), + 0n, + )).resolves.toBeUndefined(); + }); + + it("does not skip fresh owned orders whose gain is below the live fee", async () => { + const runtime = { + client: { + getTransaction: vi.fn<() => Promise<{ blockNumber: bigint }>>(() => Promise.resolve({ blockNumber: 100000n })), + }, + }; + + await expect(freshMatchableOrderSkip( + runtime as never, + [matchableOrder(byte32FromByte("46"))], + headerLike({ number: 105400n, epoch: ccc.Epoch.from([0n, 0n, 1n]) }), + ccc.fixedPointFrom(1000), + )).resolves.toBeUndefined(); + expect(runtime.client.getTransaction).not.toHaveBeenCalled(); + }); }); function matchableOrder(txHash: ccc.Hex): never { @@ -394,16 +437,32 @@ function nonMatchableOrder(txHash: ccc.Hex): never { return order(txHash, false); } -function order(txHash: ccc.Hex, isMatchable: boolean): never { +function unmarketableOrder(txHash: ccc.Hex): never { + return order(txHash, true, Ratio.from({ ckbScale: 20_000_000_000_000_000n, udtScale: 1n })); +} + +function order(txHash: ccc.Hex, isMatchable: boolean, ckbToUdt = Ratio.from({ ckbScale: 1n, udtScale: 2n })): never { + const udtScript = script("66"); + const outputData = OrderData.from({ + udtValue: 0n, + master: { type: "absolute", value: { txHash, index: 1n } }, + info: { ckbToUdt, udtToCkb: Ratio.empty(), ckbMinMatchLog: 0 }, + }).toBytes(); + const minimalCell = ccc.Cell.from({ + outPoint: { txHash, index: 0n }, + cellOutput: { lock: script("55"), type: udtScript }, + outputData, + }); return { - order: { - isMatchable: () => isMatchable, - cell: ccc.Cell.from({ - outPoint: { txHash, index: 0n }, - cellOutput: { capacity: 0n, lock: script("55") }, - outputData: "0x", - }), - }, + order: OrderCell.mustFrom(ccc.Cell.from({ + outPoint: { txHash, index: 0n }, + cellOutput: { + capacity: minimalCell.cellOutput.capacity + (isMatchable ? ccc.fixedPointFrom(100) : 0n), + lock: script("55"), + type: udtScript, + }, + outputData, + })), } as never; } diff --git a/apps/tester/src/index.ts b/apps/tester/src/index.ts index 121e936..15c24b3 100644 --- a/apps/tester/src/index.ts +++ b/apps/tester/src/index.ts @@ -27,7 +27,7 @@ import { import { freshMatchableOrderSkip } from "./freshMatchableOrderSkip.js"; const CKB = ccc.fixedPointFrom(1); const CKB_RESERVE = 2000n * CKB; -const ALL_CKB_LIMIT_ORDER_OVERHEAD = 500n * CKB; +const ALL_CKB_LIMIT_ORDER_OVERHEAD = 1000n * CKB; const MIN_TOTAL_CAPITAL_DIVISOR = 20n; const TESTER_FEE = 1n; const TESTER_FEE_BASE = 100000n; @@ -128,6 +128,7 @@ async function main(): Promise { runtime, state.userOrders, state.system.tip, + state.system.feeRate, ); if (skip) { executionLog.skip = skip; @@ -184,14 +185,17 @@ async function main(): Promise { const effectiveFeePolicy = isSdkConversionScenario(effectiveTesterScenario) ? DEFAULT_TESTER_FEE_POLICY : feePolicy; - const estimatedOrders = rawOrders.map((order) => ({ - ...order, - estimate: IckbSdk.estimate(order.direction === "ckb-to-ickb", order.amounts, state.system, { - fee: effectiveFeePolicy.fee, - feeBase: effectiveFeePolicy.feeBase, - }), - })); - if (estimatedOrders.some((order) => order.estimate.convertedAmount <= 0n)) { + const estimatedOrders: EstimatedRawOrder[] = []; + let estimateUnavailable = false; + for (const order of rawOrders) { + const estimate = estimateRawOrder(order, state.system, effectiveFeePolicy); + if (estimate === undefined) { + estimateUnavailable = true; + break; + } + estimatedOrders.push({ ...order, estimate }); + } + if (estimateUnavailable || estimatedOrders.some((order) => order.estimate.convertedAmount <= 0n)) { executionLog.skip = { reason: "estimated-conversion-too-small" }; if (logTerminalIteration(executionLog, startTime, ++completedIterations, maxIterations)) { return; @@ -498,12 +502,10 @@ function hasPositiveMultiOrderEstimates( try { const plan = planTesterTransaction(state, 0n, scenario); const orders = plannedRawOrders(plan, scenario); - return orders.length >= 2 && orders.every((order) => IckbSdk.estimate( - order.direction === "ckb-to-ickb", - order.amounts, - state.system, - { fee: feePolicy.fee, feeBase: feePolicy.feeBase }, - ).convertedAmount > 0n); + return orders.length >= 2 && orders.every((order) => { + const estimate = estimateRawOrder(order, state.system, feePolicy); + return estimate !== undefined && estimate.convertedAmount > 0n; + }); } catch (error) { if (error instanceof TesterTerminalError) { return false; @@ -560,6 +562,28 @@ function buildPlannedRawOrderTransaction( return buildRawOrderTransaction(runtime, state, rawOrders); } +function estimateRawOrder( + order: PlannedRawOrder, + system: TesterState["system"], + feePolicy: TesterFeePolicy, +): ReturnType | undefined { + try { + return IckbSdk.estimate(order.direction === "ckb-to-ickb", order.amounts, system, { + fee: feePolicy.fee, + feeBase: feePolicy.feeBase, + }); + } catch (error) { + if (isUnrepresentableTesterEstimateError(error)) { + return undefined; + } + throw error; + } +} + +export function isUnrepresentableTesterEstimateError(error: unknown): boolean { + return error instanceof Error && error.message === "Ratio scale exceeds Uint64"; +} + function orderEvidence(orders: EstimatedRawOrder[], feePolicy: TesterFeePolicy): Record { const logs = orders.map((order) => orderLog( order.direction === "ckb-to-ickb", diff --git a/packages/order/src/index.ts b/packages/order/src/index.ts index b5f5862..ef7b811 100644 --- a/packages/order/src/index.ts +++ b/packages/order/src/index.ts @@ -11,4 +11,4 @@ export { type OrderDataLike, type RelativeLike, } from "./entities.js"; -export { OrderManager, type Match, type OrderGroupSkipReason } from "./order.js"; +export { OrderManager, type Match, type MatchDiagnostics, type OrderGroupSkipReason } from "./order.js"; diff --git a/packages/order/src/order.test.ts b/packages/order/src/order.test.ts index b96c018..c71509a 100644 --- a/packages/order/src/order.test.ts +++ b/packages/order/src/order.test.ts @@ -126,6 +126,17 @@ describe("OrderMatcher", () => { expect(match.partials).toHaveLength(1); expect(match.ckbDelta).toBeLessThan(0n); expect(match.udtDelta).toBeGreaterThan(0n); + expect(match.diagnostics).toMatchObject({ + orderCount: 1, + directions: { + ckbToUdt: { matchableCount: 0 }, + udtToCkb: { matchableCount: 1 }, + }, + candidates: { + rejected: { nonPositiveGain: 0 }, + }, + }); + expect(match.diagnostics?.candidates.positiveGain).toBeGreaterThan(0); }); it("respects a partial cap when selecting the best match", () => { @@ -220,11 +231,46 @@ describe("OrderMatcher", () => { }, ); - expect(match).toEqual({ + expect(match).toMatchObject({ ckbDelta: 0n, udtDelta: 0n, partials: [], + diagnostics: { + orderCount: 1, + directions: { + ckbToUdt: { matchableCount: 0 }, + udtToCkb: { matchableCount: 1 }, + }, + candidates: { + positiveGain: 0, + }, + }, }); + expect(match.diagnostics?.candidates.rejected.nonPositiveGain).toBeGreaterThan(0); + }); + + it("reports one primary allowance rejection reason per candidate", () => { + const order = makeUdtToCkbOrder(); + + const match = OrderManager.bestMatch( + [order], + { + ckbValue: -ccc.fixedPointFrom(1000), + udtValue: -ccc.fixedPointFrom(1000), + }, + { + ckbScale: 3n, + udtScale: 5n, + }, + { + feeRate: 0n, + ckbAllowanceStep: ccc.fixedPointFrom(1), + }, + ); + + const rejected = match.diagnostics?.candidates.rejected; + expect(rejected?.insufficientCkbAllowance).toBeGreaterThan(0); + expect(rejected?.insufficientUdtAllowance).toBe(0); }); it("does not use the same order cell in both match directions", () => { diff --git a/packages/order/src/order.ts b/packages/order/src/order.ts index 1d8921f..3b8c2e3 100644 --- a/packages/order/src/order.ts +++ b/packages/order/src/order.ts @@ -303,10 +303,7 @@ export class OrderManager implements ScriptDeps { * @param options.ckbAllowanceStep - The step value for CKB allowance (defaults to 1000 CKB as fixed point). * @param options.maxPartials - Maximum matched partial outputs to keep in the result. * - * @returns A Match object containing the best combination of: - * • ckbDelta: net change in CKB, - * • udtDelta: net change in UDT, - * • partials: list of partial matches. + * @returns A Match object containing the best combination of deltas, partial matches, and search diagnostics. */ static bestMatch( orderPool: OrderCell[], @@ -346,22 +343,45 @@ export class OrderManager implements ScriptDeps { const maxPartials = options?.maxPartials; const udtAllowanceStep = (ckbAllowanceStep * ckbScale + udtScale - 1n) / udtScale; + const ckbToUdtMatchers = orderMatchers(orderPool, true, ckbMiningFee); + const udtToCkbMatchers = orderMatchers(orderPool, false, ckbMiningFee); + const diagnostics: MatchDiagnostics = { + orderCount: orderPool.length, + allowance, + ckbAllowanceStep, + udtAllowanceStep, + ckbMiningFee, + ...(maxPartials === undefined ? {} : { maxPartials }), + directions: { + ckbToUdt: summarizeMatchers(ckbToUdtMatchers), + udtToCkb: summarizeMatchers(udtToCkbMatchers), + }, + candidates: { + total: 0, + viable: 0, + positiveGain: 0, + rejected: { + maxPartials: 0, + duplicateOrder: 0, + insufficientCkbAllowance: 0, + insufficientUdtAllowance: 0, + nonPositiveGain: 0, + }, + bestGain: 0n, + }, + }; const ckb2UdtMatches = new BufferedGenerator( - OrderManager.sequentialMatcher( - orderPool, - true, + sequentialMatches( + ckbToUdtMatchers, ckbAllowanceStep, - ckbMiningFee, ), 2, ); const udt2CkbMatches = new BufferedGenerator( - OrderManager.sequentialMatcher( - orderPool, - false, + sequentialMatches( + udtToCkbMatchers, udtAllowanceStep, - ckbMiningFee, ), 2, ); @@ -389,18 +409,37 @@ export class OrderManager implements ScriptDeps { const ckbDelta = c2u.ckbDelta + u2c.ckbDelta; const udtDelta = c2u.udtDelta + u2c.udtDelta; const partials = c2u.partials.concat(u2c.partials); + diagnostics.candidates.total += 1; if (maxPartials !== undefined && partials.length > maxPartials) { + diagnostics.candidates.rejected.maxPartials += 1; continue; } if (!hasUniquePartialOrderOutPoints(partials)) { + diagnostics.candidates.rejected.duplicateOrder += 1; continue; } const ckbFee = ckbMiningFee * BigInt(partials.length); const ckbAllowance = allowance.ckbValue + ckbDelta - ckbFee; const udtAllowance = allowance.udtValue + udtDelta; const gain = (ckbDelta - ckbFee) * ckbScale + udtDelta * udtScale; + if (ckbAllowance < 0n) { + diagnostics.candidates.rejected.insufficientCkbAllowance += 1; + } else if (udtAllowance < 0n) { + diagnostics.candidates.rejected.insufficientUdtAllowance += 1; + } + if (ckbAllowance < 0n || udtAllowance < 0n) { + continue; + } + diagnostics.candidates.viable += 1; + if (partials.length > 0) { + if (gain > 0n) { + diagnostics.candidates.positiveGain += 1; + } else { + diagnostics.candidates.rejected.nonPositiveGain += 1; + } + } - if (ckbAllowance >= 0n && udtAllowance >= 0n && gain > best.gain) { + if (gain > best.gain) { best = { i, j, @@ -417,10 +456,12 @@ export class OrderManager implements ScriptDeps { } const { ckbDelta, udtDelta, partials } = best; + diagnostics.candidates.bestGain = best.gain; return { ckbDelta, udtDelta, partials, + diagnostics, }; } @@ -455,64 +496,7 @@ export class OrderManager implements ScriptDeps { allowanceStep: ccc.FixedPoint, ckbMiningFee: ccc.FixedPoint, ): Generator { - // Generate matchers from the given order pool using OrderMatcher, filter out undefined results, - // and sort the matchers by their real match ratio in decreasing order. - const matchers = orderPool - .map((o) => OrderMatcher.from(o, isCkb2Udt, ckbMiningFee)) - .filter((m) => m !== undefined) - .sort((a, b) => OrderMatcher.compareRealRatioDesc(a, b)); - - // Initialize an accumulator for the cumulative match. - let acc: Match = { - ckbDelta: 0n, - udtDelta: 0n, - partials: [], - }; - - let curr = acc; - yield curr; - - // Process each matcher in sequence. - for (const matcher of matchers) { - const maxMatch = matcher.bMaxMatch; - // Distribute maxMatch into partial matches according to a fair distribution policy: - // - Each partial match is of at least of allowanceStep size. - // - The number of partial matches is maximized. - // - The distribution is as fair as possible (i.e., partial match sizes differ by at most 1 sats). - // - // Here, N is defined as ceil(maxMatch / allowanceStep). - const N = (maxMatch + allowanceStep - 1n) / allowanceStep; - - // Determine the base quota (q) and remainder (r) for fair distribution. - // q = base units per partial match. - // r = the number of partial matches that will receive one extra unit. - const q = maxMatch / N; - const r = maxMatch % N; - - let allowance = 0n; - for (let i = 0n; i < N; i++) { - // For the first r partial matches, assign an extra unit (q + 1); for the rest, assign q. - allowance += i < r ? q + 1n : q; - - // Compute the match using the current allowance. - const m = matcher.match(allowance); - // If the current allowance is too low to yield any partial matches, - // try the next allowance for the same matcher. - if (m.partials.length === 0) { - continue; - } - // Update the cumulative match by aggregating the deltas and partials. - curr = { - ckbDelta: acc.ckbDelta + m.ckbDelta, - udtDelta: acc.udtDelta + m.udtDelta, - partials: acc.partials.concat(m.partials), - }; - // Yield the newly updated cumulative match. - yield curr; - } - // Update the accumulator with the current cumulative match for the next matcher. - acc = curr; - } + yield* sequentialMatches(orderMatchers(orderPool, isCkb2Udt, ckbMiningFee), allowanceStep); } /** @@ -850,6 +834,96 @@ function maxOrderOccupiedSize(orderPool: OrderCell[]): number { return maxSize; } +function orderMatchers( + orderPool: OrderCell[], + isCkb2Udt: boolean, + ckbMiningFee: ccc.FixedPoint, +): OrderMatcher[] { + return orderPool + .map((o) => OrderMatcher.from(o, isCkb2Udt, ckbMiningFee)) + .filter((m) => m !== undefined) + .sort((a, b) => OrderMatcher.compareRealRatioDesc(a, b)); +} + +function* sequentialMatches( + matchers: OrderMatcher[], + allowanceStep: ccc.FixedPoint, +): Generator { + // Initialize an accumulator for the cumulative match. + let acc: Match = { + ckbDelta: 0n, + udtDelta: 0n, + partials: [], + }; + + let curr = acc; + yield curr; + + // Process each matcher in sequence. + for (const matcher of matchers) { + const maxMatch = matcher.bMaxMatch; + // Distribute maxMatch into partial matches according to a fair distribution policy: + // - Each partial match is of at least of allowanceStep size. + // - The number of partial matches is maximized. + // - The distribution is as fair as possible (i.e., partial match sizes differ by at most 1 sats). + // + // Here, N is defined as ceil(maxMatch / allowanceStep). + const N = (maxMatch + allowanceStep - 1n) / allowanceStep; + + // Determine the base quota (q) and remainder (r) for fair distribution. + // q = base units per partial match. + // r = the number of partial matches that will receive one extra unit. + const q = maxMatch / N; + const r = maxMatch % N; + + let allowance = 0n; + for (let i = 0n; i < N; i++) { + // For the first r partial matches, assign an extra unit (q + 1); for the rest, assign q. + allowance += i < r ? q + 1n : q; + + // Compute the match using the current allowance. + const m = matcher.match(allowance); + // If the current allowance is too low to yield any partial matches, + // try the next allowance for the same matcher. + if (m.partials.length === 0) { + continue; + } + // Update the cumulative match by aggregating the deltas and partials. + curr = { + ckbDelta: acc.ckbDelta + m.ckbDelta, + udtDelta: acc.udtDelta + m.udtDelta, + partials: acc.partials.concat(m.partials), + }; + // Yield the newly updated cumulative match. + yield curr; + } + // Update the accumulator with the current cumulative match for the next matcher. + acc = curr; + } +} + +function summarizeMatchers( + matchers: OrderMatcher[], +): MatchDirectionDiagnostics { + const matchableCount = matchers.length; + let minAllowance: ccc.FixedPoint | undefined; + let maxMatch: ccc.FixedPoint | undefined; + for (const matcher of matchers) { + minAllowance = minAllowance === undefined || compareBigInt(matcher.bMinMatch, minAllowance) < 0 + ? matcher.bMinMatch + : minAllowance; + maxMatch = maxMatch === undefined || compareBigInt(matcher.bMaxMatch, maxMatch) > 0 + ? matcher.bMaxMatch + : maxMatch; + } + + return { + matchableCount, + ...(minAllowance === undefined ? {} : { minAllowance }), + ...(maxMatch === undefined ? {} : { maxMatch }), + }; +} + /** * Represents a partial match result for an order. */ @@ -886,6 +960,41 @@ export interface Match { */ udtOut: ccc.FixedPoint; }[]; + + /** Aggregate match-search diagnostics. It excludes order cells, scripts, and out points. */ + diagnostics?: MatchDiagnostics; +} + +export interface MatchDiagnostics { + orderCount: number; + allowance: ValueComponents; + ckbAllowanceStep: ccc.FixedPoint; + udtAllowanceStep: ccc.FixedPoint; + ckbMiningFee: ccc.FixedPoint; + maxPartials?: number; + directions: { + ckbToUdt: MatchDirectionDiagnostics; + udtToCkb: MatchDirectionDiagnostics; + }; + candidates: { + total: number; + viable: number; + positiveGain: number; + rejected: { + maxPartials: number; + duplicateOrder: number; + insufficientCkbAllowance: number; + insufficientUdtAllowance: number; + nonPositiveGain: number; + }; + bestGain: bigint; + }; +} + +export interface MatchDirectionDiagnostics { + matchableCount: number; + minAllowance?: ccc.FixedPoint; + maxMatch?: ccc.FixedPoint; } /** diff --git a/scripts/ickb-supervisor-loop.mjs b/scripts/ickb-supervisor-loop.mjs index ddb000a..1417711 100644 --- a/scripts/ickb-supervisor-loop.mjs +++ b/scripts/ickb-supervisor-loop.mjs @@ -116,12 +116,12 @@ export function summarizeRun(summary, { runIndex, relativeOutDir, status }) { export function decideNext({ run, priorOutcomes, previousSignature, stableCount, stableLimit, runIndex, maxRuns }) { const newOutcomes = run.outcomes.filter((outcome) => !priorOutcomes.has(outcome)); const nextStableCount = previousSignature === run.signature ? stableCount + 1 : 1; - if (run.status !== 0) { - return { action: "stop", reason: "supervisor_nonzero", newOutcomes, stableCount: nextStableCount, exitCode: run.status }; - } if (run.hasIncident) { return { action: "stop", reason: "incident", newOutcomes, stableCount: nextStableCount, exitCode: 2 }; } + if (run.status !== 0) { + return { action: "stop", reason: "supervisor_nonzero", newOutcomes, stableCount: nextStableCount, exitCode: run.status }; + } if (run.txCount > 0) { return { action: "stop", reason: "tx_observed", newOutcomes, stableCount: nextStableCount, exitCode: 0 }; } diff --git a/scripts/ickb-supervisor-loop.test.mjs b/scripts/ickb-supervisor-loop.test.mjs index c091c8e..c3985b0 100644 --- a/scripts/ickb-supervisor-loop.test.mjs +++ b/scripts/ickb-supervisor-loop.test.mjs @@ -121,6 +121,15 @@ test("supervisor loop decisions stop on incident, tx, new outcome, and stable no runIndex: 2, maxRuns: 10, }).reason, "supervisor_nonzero"); + assert.equal(decideNext({ + run: { ...baseRun, status: 2, hasIncident: true }, + priorOutcomes, + previousSignature: "other", + stableCount: 0, + stableLimit: 3, + runIndex: 2, + maxRuns: 10, + }).reason, "incident"); assert.equal(decideNext({ run: { ...baseRun, hasIncident: true }, priorOutcomes,