Harden live actor diagnostics and retries

apps/bot/README.md

@@ -68,11 +68,11 @@ Stable event types:
 - `bot.transaction.failed`
 - `bot.iteration.failed`
 
-No-action iterations emit `bot.decision.skipped` with `reason` and evidence. Build-time skip reasons `no_actions` and `match_value_not_above_fee` include a `decision` transcript. The pre-build safety skip `capital_below_minimum` exits with code `2` and includes zero `actions` plus `state` evidence instead of a `decision` transcript because match, rebalance, fee, and transaction shape were not evaluated. Rebalance no-op decisions include policy-owned reasons such as `insufficient_output_slots`, `low_ickb_ckb_reserve_unavailable`, `target_ickb_not_exceeded`, and `no_ready_withdrawal_selection`.
+No-action iterations emit `bot.decision.skipped` with `reason` and evidence. Build-time skip reasons `no_actions` and `match_value_not_above_fee` include a `decision` transcript. The pre-build safety skip `capital_below_minimum` exits with code `2` and includes zero `actions` plus `state` evidence instead of a `decision` transcript because match, rebalance, fee, and transaction shape were not evaluated. `bot.iteration.failed` includes a redacted `error` summary plus `retryable` and `terminal` booleans from the bot retry policy. Rebalance no-op decisions include policy-owned reasons such as `insufficient_output_slots`, `low_ickb_ckb_reserve_unavailable`, `target_ickb_not_exceeded`, and `no_ready_withdrawal_selection`.
 
 The decision transcript groups evidence under `chainTip`, `balances`, `orders`, `withdrawals`, `poolDeposits`, `match`, `rebalance`, `actions`, `fee`, `transactionShape`, `exchangeRatio`, and `depositCapacity`. Transaction events summarize action counts, fee, fee rate, tx hash, confirmation status, check count, elapsed time, and transaction shape counts.
 
-JSON `"maxIterations":1` makes `pnpm --filter ./apps/bot start` exit with code `0` after one terminal iteration when that terminal outcome is a skipped decision, committed transaction, non-safety transaction failure, or non-safety iteration failure. Safety stops still keep their nonzero behavior: low capital and confirmation timeouts after broadcast exit with code `2`. Omitting `maxIterations` keeps the default infinite loop.
+JSON `"maxIterations":1` makes `pnpm --filter ./apps/bot start` exit with code `0` after one terminal iteration when that terminal outcome is a skipped decision, committed transaction, non-safety transaction failure, or non-safety iteration failure. Retryable iteration failures are logged with `terminal:false` and do not count toward `maxIterations`. Safety stops still keep their nonzero behavior: low capital and confirmation timeouts after broadcast exit with code `2`. Omitting `maxIterations` keeps the default infinite loop.
 
 Structured events should contain public evidence and summaries needed to understand bot behavior. Do not add private keys, seed phrases, mnemonics, or other secrets to log payloads; omit noisy public fields at the call site instead of relying on redaction.
 

apps/bot/src/index.test.ts

@@ -9,7 +9,12 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { TARGET_ICKB_BALANCE } from "./policy.js";
-import { completeTerminalIteration, readBotRuntimeConfig } from "./index.js";
+import {
+  completeTerminalIteration,
+  isRetryableBotError,
+  iterationFailureEventFields,
+  readBotRuntimeConfig,
+} from "./index.js";
 import { buildTransaction, collectPoolDeposits, postTransactionPlainCkbBalance } from "./runtime.js";
 
 afterEach(() => {
@@ -163,6 +168,29 @@ describe("completeTerminalIteration", () => {
   });
 });
 
+describe("bot retryable iteration failures", () => {
+  it("treats chain-tip races and fetch transport failures as retryable", () => {
+    expect(isRetryableBotError(new Error("L1 state scan crossed chain tip; retry with a fresh state"))).toBe(true);
+    expect(isRetryableBotError(new TypeError("fetch failed"))).toBe(true);
+    expect(isRetryableBotError(new Error("fetch failed"))).toBe(false);
+    expect(isRetryableBotError(new Error("deterministic build failure"))).toBe(false);
+  });
+
+  it("emits retryability metadata from the same retry decision", () => {
+    expect(iterationFailureEventFields(new TypeError("fetch failed"))).toMatchObject({
+      retryable: true,
+      terminal: false,
+      error: { name: "TypeError", message: "fetch failed" },
+    });
+
+    expect(iterationFailureEventFields(new Error("deterministic build failure"))).toMatchObject({
+      retryable: false,
+      terminal: true,
+      error: { name: "Error", message: "deterministic build failure" },
+    });
+  });
+});
+
 describe("readBotRuntimeConfig", () => {
   it("requires a JSON config file", async () => {
     await expect(readBotRuntimeConfig({})).rejects.toThrow("Empty env BOT_CONFIG_FILE");
@@ -355,6 +383,30 @@ describe("buildTransaction", () => {
       ckbDelta: 0n,
       udtDelta: 0n,
       partials: [],
+      diagnostics: {
+        orderCount: 1,
+        allowance: { ckbValue: 0n, udtValue: 0n },
+        ckbAllowanceStep: 1n,
+        udtAllowanceStep: 1n,
+        ckbMiningFee: 1n,
+        directions: {
+          ckbToUdt: { matchableCount: 0 },
+          udtToCkb: { matchableCount: 1, minAllowance: 1n, maxMatch: 2n },
+        },
+        candidates: {
+          total: 2,
+          viable: 1,
+          positiveGain: 0,
+          rejected: {
+            maxPartials: 0,
+            duplicateOrder: 0,
+            insufficientCkbAllowance: 1,
+            insufficientUdtAllowance: 0,
+            nonPositiveGain: 1,
+          },
+          bestGain: 0n,
+        },
+      },
     });
 
     const completeTransaction = vi.fn();
@@ -369,6 +421,20 @@ describe("buildTransaction", () => {
       reason: "no_actions",
       decision: {
         rebalance: { kind: "none", reason: "target_ickb_not_exceeded" },
+        match: {
+          diagnostics: {
+            orderCount: 1,
+            directions: {
+              udtToCkb: { matchableCount: 1, minAllowance: 1n, maxMatch: 2n },
+            },
+            candidates: {
+              rejected: {
+                insufficientCkbAllowance: 1,
+                nonPositiveGain: 1,
+              },
+            },
+          },
+        },
         actions: {
           collectedOrders: 0,
           completedDeposits: 0,

apps/bot/src/index.ts

@@ -19,6 +19,7 @@ import {
   sleep,
   STOP_EXIT_CODE,
   type RuntimeConfig,
+  type SecretRedactionContext,
   verifyChainPreflight,
 } from "@ickb/node-utils";
 import {
@@ -149,10 +150,9 @@ async function main(): Promise<void> {
       }
     } catch (error) {
       stopAfterLog = handleLoopError(executionLog, error, secrets);
-      events.emit(iterationId, "bot.iteration.failed", {
-        error: errorSummary(error, secrets),
-      });
-      if (isRetryableBotError(error)) {
+      const failure = iterationFailureEventFields(error, secrets);
+      events.emit(iterationId, "bot.iteration.failed", failure);
+      if (failure.retryable) {
         logExecution(executionLog, startTime);
         continue;
       }
@@ -187,6 +187,19 @@ export function completeTerminalIteration(
   };
 }
 
+export function iterationFailureEventFields(error: unknown, secrets: SecretRedactionContext = {}): {
+  error: Record<string, unknown> | string;
+  retryable: boolean;
+  terminal: boolean;
+} {
+  const retryable = isRetryableBotError(error);
+  return {
+    error: errorSummary(error, secrets),
+    retryable,
+    terminal: !retryable,
+  };
+}
+
 async function readBotState(runtime: Runtime): Promise<BotState> {
   const accountLocks = await signerAccountLocks(runtime.signer, runtime.primaryLock);
   const { system, user, account } = await runtime.sdk.getL1AccountState(
@@ -243,8 +256,11 @@ function outPointKey(outPoint: ccc.OutPoint): string {
 
 const fmtCkb = formatCkb;
 
-function isRetryableBotError(error: unknown): boolean {
-  return error instanceof Error && error.message.includes("L1 state scan crossed chain tip");
+export function isRetryableBotError(error: unknown): boolean {
+  return error instanceof Error && (
+    error.message.includes("L1 state scan crossed chain tip") ||
+    (error instanceof TypeError && error.message === "fetch failed")
+  );
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {

apps/bot/src/runtime.ts

@@ -7,6 +7,8 @@ import {
 } from "@ickb/core";
 import {
   OrderManager,
+  type Match,
+  type MatchDiagnostics,
   type OrderCell,
   type OrderGroup,
 } from "@ickb/order";
@@ -126,6 +128,7 @@ export interface BotDecisionTranscript {
     ckbDelta: bigint;
     udtDelta: bigint;
     value?: bigint;
+    diagnostics?: MatchDiagnostics;
   };
   rebalance: {
     kind: RebalancePlan["kind"];
@@ -397,7 +400,7 @@ function buildDecisionTranscript({
   tx,
 }: {
   state: BotState;
-  match: { partials: readonly unknown[]; ckbDelta: bigint; udtDelta: bigint };
+  match: Pick<Match, "partials" | "ckbDelta" | "udtDelta" | "diagnostics">;
   rebalance: RebalancePlan;
   outputSlots: number;
   actions: BotActions;
@@ -410,6 +413,7 @@ function buildDecisionTranscript({
       partialCount: match.partials.length,
       ckbDelta: match.ckbDelta,
       udtDelta: match.udtDelta,
+      ...(match.diagnostics === undefined ? {} : { diagnostics: match.diagnostics }),
     },
     rebalance: rebalanceSummary(rebalance, outputSlots, state, match),
     actions,

apps/supervisor/src/index.test.ts

@@ -961,6 +961,17 @@ describe("classification", () => {
       }),
       botEvent("bot.transaction.built", {
         actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 1, withdrawalRequests: 0, withdrawals: 0 },
+        decision: {
+          match: {
+            diagnostics: {
+              directions: {
+                ckbToUdt: { matchableCount: 5 },
+                udtToCkb: { matchableCount: 6 },
+              },
+              candidates: { viable: 7, positiveGain: 8 },
+            },
+          },
+        },
       }),
       botEvent("bot.transaction.committed", { txHash: txHash("33"), status: "committed" }),
     ].map(JSON.stringify).join("\n");
@@ -972,12 +983,66 @@ describe("classification", () => {
       marketOrderCount: 4,
       userOrderCount: 0,
       receiptCount: 1,
+      ckbToUdtMatchableOrderCount: 5,
+      udtToCkbMatchableOrderCount: 6,
+      viableMatchCandidateCount: 7,
+      positiveGainMatchCandidateCount: 8,
       readyPoolDepositCount: 2,
       nearReadyPoolDepositCount: 1,
       futurePoolDepositCount: 3,
     });
   });
 
+  it("keeps match diagnostics tied to the matching state-read iteration", () => {
+    const stdout = [
+      botEvent("bot.state.read", {
+        iterationId: 1,
+        orders: { marketCount: 4, userCount: 0, receiptCount: 1 },
+        poolDeposits: { readyCount: 2, nearReadyCount: 1, futureCount: 3 },
+      }),
+      botEvent("bot.transaction.built", {
+        iterationId: 1,
+        actions: { collectedOrders: 0, completedDeposits: 0, matchedOrders: 1, deposits: 1, withdrawalRequests: 0, withdrawals: 0 },
+        decision: {
+          match: {
+            diagnostics: {
+              directions: {
+                ckbToUdt: { matchableCount: 5 },
+                udtToCkb: { matchableCount: 6 },
+              },
+              candidates: { viable: 7, positiveGain: 8 },
+            },
+          },
+        },
+      }),
+      botEvent("bot.state.read", {
+        iterationId: 2,
+        orders: { marketCount: 9, userCount: 1, receiptCount: 0 },
+        poolDeposits: { readyCount: 0, nearReadyCount: 0, futureCount: 0 },
+      }),
+      botEvent("bot.iteration.failed", {
+        iterationId: 2,
+        retryable: true,
+        terminal: false,
+        error: { name: "TypeError", message: "fetch failed" },
+      }),
+    ].map(JSON.stringify).join("\n");
+    const classification = classifyActorResult("bot", commandResult("bot", stdout));
+
+    expect(classification.publicState).toEqual({
+      marketOrderCount: 9,
+      userOrderCount: 1,
+      receiptCount: 0,
+      ckbToUdtMatchableOrderCount: undefined,
+      udtToCkbMatchableOrderCount: undefined,
+      viableMatchCandidateCount: undefined,
+      positiveGainMatchCandidateCount: undefined,
+      readyPoolDepositCount: 0,
+      nearReadyPoolDepositCount: 0,
+      futurePoolDepositCount: 0,
+    });
+  });
+
   it("rejects committed bot evidence without a valid tx hash", () => {
     const stdout = [
       botEvent("bot.transaction.built", {
@@ -1152,6 +1217,11 @@ describe("classification", () => {
     expect(safeArtifactText(JSON.stringify({ witnesses: ["0xsignature"], inputs: [] }))).toBe(
       "<redacted: transaction-shaped output withheld by supervisor>\n",
     );
+    expect(safeArtifactText(JSON.stringify({
+      transactionShape: { inputs: 1, outputs: 2, cellDeps: 3, headerDeps: 4, witnesses: 5 },
+    }))).toBe(
+      JSON.stringify({ transactionShape: { inputs: 1, outputs: 2, cellDeps: 3, headerDeps: 4, witnesses: 5 } }),
+    );
     expect(safeArtifactText(JSON.stringify({ system: { tip: { hash: txHash("aa") } } }))).toBe(
       JSON.stringify({ system: { tip: { hash: txHash("aa") } } }),
     );

apps/supervisor/src/index.ts

@@ -150,6 +150,10 @@ export interface PublicStateAssumption {
   marketOrderCount?: number;
   userOrderCount?: number;
   receiptCount?: number;
+  ckbToUdtMatchableOrderCount?: number;
+  udtToCkbMatchableOrderCount?: number;
+  viableMatchCandidateCount?: number;
+  positiveGainMatchCandidateCount?: number;
   readyPoolDepositCount?: number;
   nearReadyPoolDepositCount?: number;
   futurePoolDepositCount?: number;
@@ -1693,20 +1697,38 @@ function latestBotActions(records: Record<string, unknown>[]): ActionCounts | un
 }
 
 function latestPublicState(records: Record<string, unknown>[]): PublicStateAssumption | undefined {
+  const matchDiagnosticsByIteration = new Map<number, Record<string, unknown>>();
   for (let index = records.length - 1; index >= 0; index -= 1) {
     const record = records[index];
     if (record === undefined) {
       continue;
     }
-    if (stringField(record, "type") !== "bot.state.read") {
+    const type = stringField(record, "type");
+    const iterationId = numberField(record, "iterationId");
+    if (iterationId !== undefined && (type === "bot.decision.skipped" || type === "bot.transaction.built")) {
+      const matchDiagnostics = optionalRecordField(optionalRecordField(recordField(record, "decision"), "match"), "diagnostics");
+      if (matchDiagnostics !== undefined && !matchDiagnosticsByIteration.has(iterationId)) {
+        matchDiagnosticsByIteration.set(iterationId, matchDiagnostics);
+      }
+    }
+    if (type !== "bot.state.read") {
       continue;
     }
     const orders = recordField(record, "orders");
     const poolDeposits = recordField(record, "poolDeposits");
+    const latestMatchDiagnostics = iterationId === undefined ? undefined : matchDiagnosticsByIteration.get(iterationId);
+    const directions = optionalRecordField(latestMatchDiagnostics, "directions");
+    const ckbToUdt = optionalRecordField(directions, "ckbToUdt");
+    const udtToCkb = optionalRecordField(directions, "udtToCkb");
+    const candidates = optionalRecordField(latestMatchDiagnostics, "candidates");
     return {
       marketOrderCount: numberField(orders, "marketCount"),
       userOrderCount: numberField(orders, "userCount"),
       receiptCount: numberField(orders, "receiptCount"),
+      ckbToUdtMatchableOrderCount: numberField(ckbToUdt, "matchableCount"),
+      udtToCkbMatchableOrderCount: numberField(udtToCkb, "matchableCount"),
+      viableMatchCandidateCount: numberField(candidates, "viable"),
+      positiveGainMatchCandidateCount: numberField(candidates, "positiveGain"),
       readyPoolDepositCount: numberField(poolDeposits, "readyCount"),
       nearReadyPoolDepositCount: numberField(poolDeposits, "nearReadyCount"),
       futurePoolDepositCount: numberField(poolDeposits, "futureCount"),
@@ -1765,7 +1787,7 @@ function containsSecretLeak(text: string): boolean {
 }
 
 function containsTransactionLeak(text: string): boolean {
-  return /["']?(witnesses|cellDeps|headerDeps|inputs|outputs|outputsData)["']?\s*:/iu.test(text);
+  return /["']?(witnesses|cellDeps|headerDeps|inputs|outputs|outputsData)["']?\s*:\s*\[/iu.test(text);
 }
 
 export function safeArtifactText(text: string): string {
@@ -1826,6 +1848,10 @@ function recordField(record: Record<string, unknown>, key: string): Record<strin
   return isRecord(value) ? value : undefined;
 }
 
+function optionalRecordField(record: Record<string, unknown> | undefined, key: string): Record<string, unknown> | undefined {
+  return record === undefined ? undefined : recordField(record, key);
+}
+
 function stringField(record: Record<string, unknown> | undefined, key: string): string | undefined {
   const value = record?.[key];
   return typeof value === "string" ? value : undefined;