From 8dc25d286f304713a7cc07afdcdc1013e0e93637 Mon Sep 17 00:00:00 2001 From: wangzhao Date: Thu, 23 Jul 2026 23:55:49 +0800 Subject: [PATCH 1/2] fix: allow extra exec deny patterns via config Add tools.exec.extra_deny_patterns for host-specific deny regexes when running un-sandboxed; product default deny list unchanged. Co-authored-by: Claude (claude-opus-4-8) --- raven/agent/loop/main.py | 1 + raven/agent/subagent/manager.py | 1 + raven/agent/tools/shell.py | 8 +++++++ raven/config/schema.py | 4 ++++ tests/test_sandbox_unit.py | 42 +++++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+) diff --git a/raven/agent/loop/main.py b/raven/agent/loop/main.py index 7776f29..3c097cf 100644 --- a/raven/agent/loop/main.py +++ b/raven/agent/loop/main.py @@ -580,6 +580,7 @@ def _register_default_tools(self) -> None: restrict_to_workspace=self.restrict_to_workspace, path_append=self.exec_config.path_append, executor=self._executor, + extra_deny_patterns=self.exec_config.extra_deny_patterns, ) ) self.tools.register(WebSearchTool(api_key=self.brave_api_key, proxy=self.web_proxy)) diff --git a/raven/agent/subagent/manager.py b/raven/agent/subagent/manager.py index 93f938d..a2d5194 100644 --- a/raven/agent/subagent/manager.py +++ b/raven/agent/subagent/manager.py @@ -166,6 +166,7 @@ async def _run_subagent_inner( restrict_to_workspace=self.restrict_to_workspace, path_append=self.exec_config.path_append, executor=executor, + extra_deny_patterns=self.exec_config.extra_deny_patterns, ) ) tools.register(WebSearchTool(api_key=self.brave_api_key, proxy=self.web_proxy)) diff --git a/raven/agent/tools/shell.py b/raven/agent/tools/shell.py index 147462c..1ad003d 100644 --- a/raven/agent/tools/shell.py +++ b/raven/agent/tools/shell.py @@ -26,6 +26,7 @@ def __init__( restrict_to_workspace: bool = False, path_append: str = "", executor: SandboxExecutor | None = None, + extra_deny_patterns: list[str] | None = None, ): self.timeout = timeout self.working_dir = working_dir @@ -40,6 +41,13 @@ def __init__( r"\b(shutdown|reboot|poweroff)\b", # system power r":\(\)\s*\{.*\};\s*:", # fork bomb ] + # Operator-configurable extras (tools.exec.extra_deny_patterns), appended + # to the built-in defaults; empty by default so product behaviour is + # unchanged. The proactivity-eval harness sets these to block host GUI + # automation (osascript / `open -a|-b`) because it runs the agent + # un-sandboxed on the operator's machine — not a product default. + if extra_deny_patterns: + self.deny_patterns = self.deny_patterns + list(extra_deny_patterns) self.allow_patterns = allow_patterns or [] self.restrict_to_workspace = restrict_to_workspace self.path_append = path_append diff --git a/raven/config/schema.py b/raven/config/schema.py index 6da93ce..7fc7170 100644 --- a/raven/config/schema.py +++ b/raven/config/schema.py @@ -489,6 +489,10 @@ class ExecToolConfig(Base): timeout: int = 60 path_append: str = "" + # Extra regex deny-patterns appended to ExecTool's built-in destructive-command + # defaults. Empty by default. Operators (or eval harnesses running the agent + # un-sandboxed) can add host-specific blocks, e.g. osascript / `open -a`. + extra_deny_patterns: list[str] = Field(default_factory=list) class MediaToolConfig(Base): diff --git a/tests/test_sandbox_unit.py b/tests/test_sandbox_unit.py index 5d050fe..d94f9c6 100644 --- a/tests/test_sandbox_unit.py +++ b/tests/test_sandbox_unit.py @@ -297,6 +297,48 @@ async def test_non_sandboxed_deny_list_runs(self, tmp_path): assert "blocked" in result assert len(executor.calls) == 0 + # Host GUI automation (osascript / `open -a|-b`) is NOT a product default — + # it is opt-in via extra_deny_patterns (the proactivity-eval harness sets it + # because it runs the agent un-sandboxed on the operator's machine). + _GUI_DENY = [r"\bosascript\b", r"\bopen\s+-[ab]\b"] + + async def test_extra_deny_patterns_block_host_gui_automation(self, tmp_path): + """With extra_deny_patterns set, osascript / `open -a|-b` are blocked + (non-sandboxed path), while opening a file and benign commands run.""" + from raven.agent.tools.shell import ExecTool + + async def run(cmd): + return await ExecTool( + executor=DirectMockExecutor(), + working_dir=str(tmp_path), + extra_deny_patterns=self._GUI_DENY, + ).execute(cmd) + + for cmd in ( + "osascript -e 'tell application \"Music\" to play'", + "open -a Music", + "open -b com.apple.Music", + ): + assert "blocked" in await run(cmd), f"should block: {cmd}" + + for cmd in ("open notes.txt", "echo hi", "ls -la"): + assert "blocked" not in await run(cmd), f"should allow: {cmd}" + + # Known accepted collateral: the security-broad ``\bosascript\b`` also + # trips when 'osascript' is a mere argument. Pinned so a future narrowing + # to command-position is a deliberate change, not an accident. + assert "blocked" in await run("grep osascript /var/log/system.log") + + async def test_gui_automation_not_blocked_by_product_default(self, tmp_path): + """Product default (no extra_deny_patterns): osascript is NOT blocked — + the GUI-automation block is eval-scoped, not shipped behaviour.""" + from raven.agent.tools.shell import ExecTool + + executor = DirectMockExecutor() + result = await ExecTool(executor=executor, working_dir=str(tmp_path)).execute("osascript -e x") + assert "blocked" not in result + assert len(executor.calls) == 1 + async def test_path_append_sandboxed_injects_export(self, tmp_path): """path_append with sandboxed executor: wraps command with export PATH.""" from raven.agent.tools.shell import ExecTool From 1388abf7762b464fd5903828c793813596f1ded8 Mon Sep 17 00:00:00 2001 From: wangzhao Date: Thu, 23 Jul 2026 23:56:05 +0800 Subject: [PATCH 2/2] chore: longrun v3 matched-model rerun and eval harness updates All agents on openrouter qwen3.5-27b; add hermes CLI pbench runner, uniform prompts, run_meta/soft_dnd stamps; refresh READMEs to v3. Co-authored-by: Claude (claude-opus-4-8) --- benchmarks/proactivity_eval/README.md | 102 +-- .../proactivity_eval/data/longrun/README.md | 51 +- .../data/longrun/README_zh.md | 42 +- .../runners/_common/backends/raven.py | 149 +++- .../runners/_common/drivers/longrun.py | 17 +- .../runners/_common/drivers/pbench.py | 12 +- .../runners/_common/longrun_adapters.py | 426 +++++++++++- .../runners/_common/mcp_cron_server.mjs | 74 +- .../runners/_common/user_simulator.py | 16 + .../runners/hermes_cli_pbench.py | 143 ++++ .../runners/longrun_scorecard.py | 641 ++++++++++++++++-- .../prompts/uniform/uniform_agent.yaml | 25 + benchmarks/proactivity_eval/runners/run.py | 16 + 13 files changed, 1504 insertions(+), 210 deletions(-) create mode 100644 benchmarks/proactivity_eval/runners/hermes_cli_pbench.py create mode 100644 benchmarks/proactivity_eval/runners/prompts/uniform/uniform_agent.yaml diff --git a/benchmarks/proactivity_eval/README.md b/benchmarks/proactivity_eval/README.md index fdfee45..32dc688 100644 --- a/benchmarks/proactivity_eval/README.md +++ b/benchmarks/proactivity_eval/README.md @@ -12,7 +12,6 @@ ``` proactivity_eval/ ├── README.md -├── FINDINGS-v12.md 历史 baseline 数据（旧版本对照） ├── runners.config.yaml 系统/agent 路径与 provider 默认 ├── data/ │ ├── pbench/test_data.jsonl pbench 输入（ProactiveAgent reward_data S1 protocol, vendored） @@ -22,7 +21,7 @@ proactivity_eval/ │ ├── _common/ backends + drivers + shared helpers │ ├── agents/{raven,hermes,openclaw}/ per-agent config + adapter glue │ ├── benchmarks/{pbench,longrun}/ per-benchmark config -│ ├── prompts/ pbench 各 agent 模板 +│ ├── prompts/ pbench 模板（统一最小模板：prompts/uniform/） │ ├── pa_scorecard.py / longrun_scorecard.py 聚合脚本 │ └── README.md runner 用法 └── output/ JSON + scorecard @@ -31,101 +30,116 @@ proactivity_eval/ ## 实验结果 ### Agent 版本 -| Agent | Version / 包版本 | Date | + +两张结果表均为 latest 版本实测（pbench 于 2026-07-22 用统一 prompt 重测；longrun 于 2026-07-23 用新 harness + 新计分器全量重跑，三家统一走 OpenRouter qwen3.5-27b）： + +| Agent | pbench 主表（2026-07-22，latest） | longrun（2026-07-23，latest） | |---|---|---| -| **Raven** | `raven 0.1.0` | 2026-04-28 | -| **Hermes** | `hermes-agent 0.10.0` | 2026-04-18 | -| **OpenClaw** | `openclaw 2026.2.1` | 2026-02-03 | +| **Raven** | `raven 0.1.2` | `raven 0.1.2` | +| **Hermes** | `hermes-agent 0.19.0` @ `9eb7b1a6`（2026-07-20，dev install） | `hermes-agent 0.19.0` @ `9eb7b1a6` | +| **OpenClaw** | `openclaw 2026.6.34`（docker，ghcr digest `25f5bacf5174…`） | `openclaw 2026.6.34`（docker） | + ### 结果 -**pbench** (N=120 reward_data)：单轮"该不该 surface help"决策，同 backend qwen3.5-27B。 +**pbench** (N=120 reward_data)：单轮"该不该 surface help"决策。所有 agent 使用**统一最小任务 prompt**（无人设、无决策原则，模板见 `runners/prompts/uniform/`），各自 CLI 黑盒调用，同 backend qwen3.5-27b（OpenRouter）。 -| Agent | TP | FP | TN | FN | Precision | Recall | F1 | mean/record | -|---|---:|---:|---:|---:|---:|---:|---:|---:| -| **Raven** | 27 | 4 | 47 | 42 | 0.871 | 0.391 | **0.540** | 11.0s | -| **Hermes** (v0.10) | 10 | 0 | 51 | 59 | **1.000** | 0.145 | 0.253 | 17.1s | -| **OpenClaw** | 10 | 0 | 51 | 59 | **1.000** | 0.145 | 0.253 | 24.7s | +| Agent | 调用方式 | TP/FP/TN/FN¹ | Precision | Recall | F1（3 runs mean） | mean/record | +|---|---|---:|---:|---:|---:|---:| +| **Raven** (0.1.2) | `raven agent --message` | 48.0/17.3/33.7/21.0 | 0.735 | **0.696** | **0.715**（0.707–0.725） | 19.4s | +| **OpenClaw** (2026.6.34) | `openclaw agent --local`（docker） | 44.0/16.0/35.0/25.0 | 0.733 | 0.638 | 0.682（0.641–0.722） | 15.4s | +| **Hermes** (v0.19.0) | `hermes chat -q -Q` | 33.7/13.3/37.7/35.3 | 0.717 | 0.488 | 0.579（0.545–0.628） | 25.5s | -Hermes / OpenClaw 完美 `Precision=1.000` 是**架构性保守**——只在 very confident 时说 yes；代价是 `Recall=0.145` 远低于 Raven 的 0.391 +¹ 三次独立运行的均值（括号内为 F1 逐轮范围），2026-07-22 实测。分数是**版本快照**，不代表最新版本能力。 -**longrun** (6 persona × 30 day)：跨日 anticipatory proactivity，同 backend qwen3.5-27B + +**longrun** (6 persona × 30 day)：跨日 anticipatory proactivity，同 backend qwen3.5-27b（OpenRouter） | 能力维度 | Raven | Hermes | OpenClaw | 含义 | |---|---|---|---|---| -| **Anticipatory** ⭐
(rubric Type A 命中) | **19/43 (44%)** | 0/43 | 0/43 | "agent 没被告知就想到该做"——只有 L3 Sentinel 能做 | -| **Scheduled execution**
(delivered **cron** fires, trajectory-derived)³ | **109 fires**
(+155 sentinel anticipatory) | 115 fires
(原生 cron) | 61 fires²
(MCP-gateway) | user 显式说"X 时提醒"后 agent 是否真的注册并 fire | -| **Reactive Q&A**
(rubric Type B 命中) | 15/21 (71%) | **18/21 (86%)** | 11/21 (52%) | user 问问题时 agent 答对率 | -| **Restraint** 🛑
(rubric Type C 命中)¹ | 10/21 (48%)
31/62 pts | **16/21 (76%)**
49/62 pts | **16/21 (76%)**
49/62 pts | DND / 频率 / 周末 constraint 是否被破坏（不该 fire 时是否克制） | +| **Anticipatory** ⭐
(rubric Type A 命中) | **19/43 (44%)** | 1/43 (2%) | 1/43 (2%) | "agent 没被告知就想到该做"——自主注册预判 / Sentinel 主动 surface | +| **Scheduled execution**
(delivered **cron** fires, trajectory-derived)² | 109 fires
(+87 sentinel anticipatory) | 103 fires
(原生 cron)³ | **267 fires**
(MCP-gateway, repeat)¹ | user 显式说"X 时提醒"后 agent 是否真的注册并 fire | +| **Reactive Q&A**
(rubric Type B 命中) | 6/21 (29%) | **10/21 (48%)** | 5/21 (24%) | user 问问题时 agent 答对率 | +| **Restraint** 🛑
(rubric Type C 命中) | 12/21 (57%) | **17/21 (81%)**| 16/21 (76%) | DND / 频率 / 周末 constraint 是否被破坏（不该 fire 时是否克制） | + +¹ OpenClaw 经捆绑的 MCP cron server（gateway 模式，详见 [`data/longrun/README.md`](data/longrun/README.md#3-openclaw-reactive--mcp-gateway-cron-baseline)）注入 `set_reminder` 工具后注册并触发 cron。桥现已支持 `repeat`（daily/weekdays/weekly）。 -¹ longrun 结果数据源（均以当前 `longrun_scorecard.py` 同版重打分，确保口径一致）；同 backend qwen3.5-27B。 +² **Scheduled execution 只计 cron fire**。衡量"用户显式预约的提醒是否真的注册并触发"，是 cron 的职责；Raven 的 sentinel anticipatory fire 属于 Anticipatory 维，作旁注 `(+N sentinel)` 显示但**不计入**本行。三家分别 109 / 103 / 267 fire（此维由 caregiver 每日服药提醒主导，且受 recurring vs one-shot 注册风格影响，波动大；OC 因 MCP-gateway repeat 明显偏高），说明三家都能可靠触发用户预约——**不作能力排名**。 -² OpenClaw 经捆绑的 MCP cron server（gateway 模式，详见 [`data/longrun/README.md`](data/longrun/README.md#3-openclaw-reactive--mcp-gateway-cron-baseline)）注入 `set_reminder` 工具后能注册并触发 cron（61 fires）。OC 的 cron 为**一次性**（非 recurring），需 agent 每天重新注册；以 caregiver（每日 3 药）为例，OC 前 ~15 天逐日 re-arm，此后停止重新注册、cron fire 干涸（trajectory 仍为完整 30 天、对话正常）。对照 Hermes 把 3 药注册为 recurring job，05-27~30 无对话仍照常 fire。OC 的 61 是其一次性模型下未能维持循环提醒的**欠发下界，非克制**。 +³ **Hermes 的 suggestions 通道在本 harness 无得分路径**：v0.19 的建议投递是 pull-only（用户主动 pull `hermes suggestions` 才可见）。harness 以 `cron_suggestion` 事件记录每条生成的建议（observability，不代投递、不计分），数量单独统计。 **OpenClaw 嵌入模式的 session 卫生由 harness 代行**：`agent --local` 不做 compaction 也不轮换 session，固定 session 跑 30 天在 2026.6.34 上第 5 天即撞上下文墙（"Context overflow"，其后每轮报错）。harness 按其**产品自身默认**（`session_reset: at_hour 4`）每模拟日轮换 session，跨日记忆走其 workspace MEMORY.md bootstrap。 -³ **Scheduled execution 只计 cron fire**（`longrun_scorecard.py::_count_delivered_fires` 的 `total = cron`）。这一维衡量"用户显式预约的提醒是否真的注册并触发"，是 cron 的职责；Raven 的 sentinel anticipatory fire 属于 Anticipatory 维，作旁注 `(+N sentinel)` 显示但**不计入**本行。 **四个维度反映的差距：** -- **Anticipatory**：EC 19 vs Hermes 0 vs OC 0 是**架构级差距**——L3 Sentinel 是唯一能在用户没显式提示时主动 surface 的层。Hermes/OC 永远是 0（cron 不管原生还是 MCP-gateway，都只能执行已注册的 job，不能"预知"）。 -- **Scheduled execution**（cron fire，见 ³）：EC 109 / Hermes 115 / OC 61。整个 longrun cron 应有量 ≈ 100–110，EC 的 109 落在合理区间，OC 61 偏低（欠注册）。EC 另有 155 次 sentinel anticipatory fire，归在 Anticipatory 维、不计入本行。 -- **Reactive Q&A**：EC 15/21 (71%) / Hermes 18/21 (86%) / OC 11/21 (52%)。同 backend qwen-27B 下 Hermes 最稳；OC 在多个 persona 的问答上回退（freelancer 0/4、parent 2/4）。 -- **Restraint**：EC 10/21 vs Hermes 16/21 vs OC 16/21。**Anticipatory 和 Restraint 是同一硬币的两面**——会主动 fire 的 agent 同时也更容易撞到 quiet_hours / bedtime / 周末 窗口。Hermes / OC 拿到接近满分（16/21）的代价是 **Anticipatory=0**——这是 "几乎不主动 ⇒ 几乎不违例" 的廉价分，**只看 Restraint 单维会奖励 always-hold agent**，必须和 Anticipatory / Scheduled execution 联合判读。 +- **Anticipatory**：Raven 19/43 (44%) vs Hermes 1/43 (2%) vs OC 1/43 (2%)——在决策归属计分（用户预约的提醒全部排除）、sonnet 强法官下，Raven 有领先的主动提醒倾向。 +- **Scheduled execution**（cron fire）：Raven 109 / Hermes 103 / OC 267——三家都能可靠注册并触发用户预约的提醒；此维由 caregiver 每日服药提醒主导、且受 recurring vs one-shot 风格影响，波动大，**不作能力排名**（见脚注 ²）。 +- **Reactive Q&A**：Raven 6/21 (29%) / Hermes 10/21 (48%) / OC 5/21 (24%)，sonnet judge 打分。 +- **Restraint**：Raven 12/21 (57%) vs Hermes 17/21 (81%) vs OC 16/21 (76%)。**Anticipatory 和 Restraint 是同一硬币的两面**——越主动，越可能在用户不期望的时间打扰到用户（Raven 主动性最高，故 restraint 最低）。 ## 用法 +> 下方命令均从 repo 根目录运行；`` = 输出目录（默认 `benchmarks/proactivity_eval/output/`）。 + ### pbench ```bash -# Smoke (n=10 stratified, ~3 min) +# smoke（n=10 分层抽样，~3min），默认统一最小 prompt（结果表口径） uv run python benchmarks/proactivity_eval/runners/run.py \ --agent raven --benchmark pbench --n 10 --context-mode cold \ - --output benchmarks/proactivity_eval/output/pbench-smoke.json + --output /pbench-smoke.json -# Full (n=120, ~30-40 min) +# 全量（n=120，~30-40min） uv run python benchmarks/proactivity_eval/runners/run.py \ --agent raven --benchmark pbench --n 120 --context-mode cold \ - --output benchmarks/proactivity_eval/output/pbench-n120.json + --output /pbench-n120.json + +# hermes 走自带 CLI（结果表口径的 hermes 通道） +uv run python benchmarks/proactivity_eval/runners/hermes_cli_pbench.py \ + --n 120 --concurrency 6 \ + --home-template <含 config.yaml/.env/auth.json 的目录> \ + --output /pbench-hermes-cli.json -# Scoring → markdown table +# 旧的各 agent 人设模板仅用于复现消融附表：加 --prompts-dir benchmarks/proactivity_eval/runners/prompts + +# 打分 → markdown 表 uv run python benchmarks/proactivity_eval/runners/pa_scorecard.py \ - --ec-agent-cold benchmarks/proactivity_eval/output/pbench-n120.json \ - --output benchmarks/proactivity_eval/output/pbench-n120-scorecard.md + --ec-agent-cold /pbench-n120.json \ + --output /pbench-n120-scorecard.md ``` ### longrun -`run.py` writes trajectories to `output/longrun/` by default, and -`longrun_scorecard.py` reads/writes the same dir by default. To score a -snapshot stored elsewhere (e.g. `output/post-C-cleanup-d30/`), pass -`--output-dir` to the scorecard — no need to move files. (`run.py` has -its own `--output-dir` for choosing where trajectories land.) +`run.py` 默认把轨迹写到 `/longrun/`，`longrun_scorecard.py` 默认读写同一目录。 +要对存放在别处的快照打分，给 scorecard 传 `--output-dir` 即可，无需搬文件 +（`run.py` 也有自己的 `--output-dir`，决定轨迹落盘位置）。 ```bash -# Smoke (single persona, 1 day, ~5 min) — output → output/longrun/ by default +# smoke（单 persona，1 天，~5min），默认输出到 /longrun/ uv run python benchmarks/proactivity_eval/runners/run.py \ --agent raven --benchmark longrun --case parent-01 --day-limit 1 -# Full (all 6 personas × 30 days; expect hours) +# 全量（6 persona × 30 天，耗时数小时） uv run python benchmarks/proactivity_eval/runners/run.py \ --agent raven --benchmark longrun --all -# Score one persona×agent (reads output/longrun/longrun---trajectory.jsonl) + +# 单个 persona×agent 打分（读 /longrun/longrun---trajectory.jsonl） uv run python benchmarks/proactivity_eval/runners/longrun_scorecard.py \ --persona parent-01 --agent raven -# Score all personas + per-persona comparison + 跨 persona × agent capability 表 -# (产出本 README 上面那张 longrun 结果表的 markdown：output/longrun/aggregate-scorecard.md) +# 全部 persona 打分 + 逐 persona 对比 + 跨 persona×agent 能力表 +# （产出本 README 上面那张 longrun 结果表：/longrun/aggregate-scorecard.md） uv run python benchmarks/proactivity_eval/runners/longrun_scorecard.py \ --all --compare --aggregate # 只重新生成 aggregate 表（已有 *-scorecard.json 时不重跑评分） uv run python benchmarks/proactivity_eval/runners/longrun_scorecard.py --aggregate -# Score / aggregate a snapshot in a non-default dir (no file moving) +# 对非默认目录的快照打分/聚合（无需搬文件） uv run python benchmarks/proactivity_eval/runners/longrun_scorecard.py \ - --aggregate --output-dir benchmarks/proactivity_eval/output/post-C-cleanup-d30 + --aggregate --output-dir / ``` 详见 `runners/README.md`。 diff --git a/benchmarks/proactivity_eval/data/longrun/README.md b/benchmarks/proactivity_eval/data/longrun/README.md index 1138d76..857be80 100644 --- a/benchmarks/proactivity_eval/data/longrun/README.md +++ b/benchmarks/proactivity_eval/data/longrun/README.md @@ -137,7 +137,7 @@ flowchart TB RL["RoutineLearner
mines repeated user
patterns over weeks"]:::sentinel MW["SentinelMemoryWriter
MEMORY.md · SOUL.md"]:::sentinel - VLLM[("Backing LLM
vLLM @ qwen3.5-27B
OpenAI-compatible endpoint")]:::backend + VLLM[("Backing LLM
qwen/qwen3.5-27b
OpenAI-compatible endpoint")]:::backend OUT(["📤 nudge delivered
to active user channel
(in-app / push)"]):::io @@ -286,7 +286,7 @@ You will need: | Component | Default | |---|---| -| Backing LLM | OpenAI-compatible endpoint serving **`qwen3.5-27B`** (LAN vLLM in our setup) | +| Backing LLM | OpenAI-compatible endpoint serving **`qwen/qwen3.5-27b`** (OpenRouter in the reference run; any OpenAI-compatible endpoint, incl. LAN vLLM, works) | | User simulator | OpenRouter `anthropic/claude-sonnet-4.5` (default; override with `--simulator-model`) | | Raven agent | `pip install raven==0.1.0` | | Hermes agent | `hermes-agent==0.10.0` source tree on disk (`$HERMES_AGENT_SRC`) | @@ -351,7 +351,8 @@ HERMES_AGENT_SRC=~/path/to/hermes-agent uv run python \ ### 3. OpenClaw (reactive + MCP-gateway cron baseline) -OpenClaw has no anticipatory layer, so it still scores 0 on Type A. But +OpenClaw has no anticipatory layer, so it scores ~0 on Type A (1/43 — a +single autonomous cron that happened to match; no L3 prediction). But since OC ≥ 2026.3.31 supports MCP, the harness bundles a Node-based MCP cron server (`runners/_common/mcp_cron_server.mjs`) and pre-wires it into the per-persona `OPENCLAW_HOME`. The OC agent's LLM can then call @@ -362,14 +363,14 @@ Requires the MCP-enabled image (`openclaw:local-mcp`); the legacy `openclaw:local` (2026.2.x) has no MCP support and will silently produce zero cron fires. -The MCP `set_reminder` registers **one-shot** reminders (not recurring), -so the agent must re-arm a daily reminder every day. In practice OC's LLM -re-arms the caregiver meds for the first ~15 days, then stops re-registering -and cron fires dry up — even though the conversation runs the full 30 days -(the harness keeps firing whatever is still registered). By contrast Hermes -registers the meds as recurring jobs, which keep firing on 05-27..05-30 with -no conversation at all. So OpenClaw's 61 is an *under-delivery* floor of its -one-shot model failing to sustain recurring reminders — not restraint. +The MCP `set_reminder` now supports `repeat` (daily/weekdays/weekly). In +this run OC's LLM both re-registers reminders aggressively and uses +recurring jobs, so its crons pile up: for caregiver alone, 92 registrations +firing 170 times, and 267 across all personas — well above the ~81–110 +ground-truth band. So OpenClaw's 267 is an *over*-delivery, not superior +scheduling (nor restraint). The legacy one-shot bridge, before `repeat` +support, instead *under*-fired as the agent failed to re-arm daily +reminders — either way this row is not a capability ranking. ```bash # Re-tag the upstream MCP-capable image as openclaw:local-mcp once: @@ -459,14 +460,14 @@ rubric: Per-persona total: `sum(per_outcome_points × pass_fraction) / total_points`. -### Reference results (qwen3.5-27B backend, 6 personas × 30 days) +### Reference results (qwen/qwen3.5-27b backend via OpenRouter, 6 personas × 30 days) -| Dimension | Raven | Hermes (v0.10) | OpenClaw | +| Dimension | Raven | Hermes (v0.19.0) | OpenClaw | |---|---|---|---| -| **A. Anticipatory proactivity** (Type A hits / lift) | **19/43 (44%)**, lift = 60 | 0/43, lift = 0 | 0/43, lift = 0 | -| **B. Reactive Q&A** (Type B hits) | 15/21 (71%) | **18/21 (86%)** | 11/21 (52%) | -| **D. Restraint** (Type C hits) | 10/21 (48%) | **16/21 (76%)** | **16/21 (76%)** | -| **C. Scheduled execution** (delivered **cron** fires across personas)¹ | **109** (+155 sentinel anticipatory) | 115 (all cron) | 61 (MCP-gateway) | +| **A. Anticipatory proactivity** (Type A hits / lift) | **19/43 (44%)**, lift = 59 | 1/43 (2%), lift = 4 | 1/43 (2%), lift = 3 | +| **B. Reactive Q&A** (Type B hits) | 6/21 (29%) | **10/21 (48%)** | 5/21 (24%) | +| **D. Restraint** (Type C hits) | 12/21 (57%) | **17/21 (81%)** | 16/21 (76%) | +| **C. Scheduled execution** (delivered **cron** fires across personas)¹ | **109** (+87 sentinel anticipatory) | 103 (all cron) | **267** (MCP-gateway) | ¹ Scheduled execution counts **cron fires only** (delivered scheduled reminders). Raven's sentinel anticipatory fires are shown as an aside @@ -477,12 +478,14 @@ freelancer (3 one-shots) issue explicit `set_reminder` intents; the other 4 personas issue none. Ground-truth cron for the whole longrun ≈ 81–93 (explicit requests) up to ~110 (incl. agent-derived contextual todos such as parent's kid/family items). Raven's 109 sits in that band (caregiver -87 ≈ correct, parent 18 context-derived); OpenClaw's 61 is *under*-firing -(missed reminders), not restraint. **Do not read this row as a capability -ranking** — the real differentiator is row A. OpenClaw runs via the -MCP-gateway cron path (`openclaw:local-mcp`, OC ≥ 2026.3.31); the legacy -`openclaw:local` (2026.2.x) image had no MCP and bottomed out at zero. -All three re-scored with the current `longrun_scorecard.py` for a +81 ≈ correct, parent 24 context-derived); Hermes's 103 likewise (caregiver +94). OpenClaw's 267 is *over*-firing — with the MCP bridge's `repeat` +support its recurring reminders re-fire daily and pile up (caregiver 170, +parent 78) — not superior scheduling. **Do not read this row as a +capability ranking** — the real differentiator is row A. OpenClaw runs via +the MCP-gateway cron path (`openclaw:local-mcp`, OC ≥ 2026.3.31); the +legacy `openclaw:local` (2026.2.x) image had no MCP and bottomed out at +zero. All three scored with the current `longrun_scorecard.py` for a consistent metric. These numbers are *measurements of architecture, not just of model @@ -530,7 +533,7 @@ of time, never *anticipate* what the user has yet to mention. - **No held-out test split.** All 6 personas are public. A hidden split is planned for the eventual NeurIPS-D&B-style release. - **Single backing model in reference numbers.** The reference table - reports qwen3.5-27B for all three agents. Cross-model studies (GPT-4o + reports qwen/qwen3.5-27b for all three agents. Cross-model studies (GPT-4o agent vs Claude agent vs local Qwen) are not yet in this release. --- diff --git a/benchmarks/proactivity_eval/data/longrun/README_zh.md b/benchmarks/proactivity_eval/data/longrun/README_zh.md index 8bd6fd0..7070640 100644 --- a/benchmarks/proactivity_eval/data/longrun/README_zh.md +++ b/benchmarks/proactivity_eval/data/longrun/README_zh.md @@ -132,7 +132,7 @@ flowchart TB RL["RoutineLearner
跨周挖掘
重复用户行为"]:::sentinel MW["SentinelMemoryWriter
MEMORY.md · SOUL.md"]:::sentinel - VLLM[("后端 LLM
vLLM @ qwen3.5-27B
OpenAI 兼容 endpoint")]:::backend + VLLM[("后端 LLM
qwen/qwen3.5-27b
OpenAI 兼容 endpoint")]:::backend OUT(["📤 nudge 投递
到活跃用户 channel
(in-app / push)"]):::io @@ -279,7 +279,7 @@ type_c_restraint: # 别过度 fire；尊重 quiet hours | 组件 | 默认 | |---|---| -| 后端 LLM | OpenAI 兼容 endpoint，serve **`qwen3.5-27B`**（我们这边是 LAN 内 vLLM） | +| 后端 LLM | OpenAI 兼容 endpoint，serve **`qwen/qwen3.5-27b`**（参考跑用 OpenRouter；任何 OpenAI 兼容 endpoint，含 LAN vLLM，均可） | | User simulator | OpenRouter `anthropic/claude-sonnet-4.5`（默认；`--simulator-model` 可覆盖） | | Raven agent | `pip install raven==0.1.0` | | Hermes agent | 本地有 `hermes-agent==0.10.0` 源码树（`$HERMES_AGENT_SRC`） | @@ -343,7 +343,8 @@ HERMES_AGENT_SRC=~/path/to/hermes-agent uv run python \ ### 3. OpenClaw（reactive + MCP-gateway cron 基线） -OpenClaw 没有 anticipatory 层，所以 Type A 依然归零。但从 OC ≥ +OpenClaw 没有 anticipatory 层，所以 Type A 近乎归零（1/43——仅一次凑巧命中的 +自主 cron，无 L3 预判）。但从 OC ≥ 2026.3.31 起支持 MCP，harness 把一个 Node 版的 MCP cron server （`runners/_common/mcp_cron_server.mjs`）预装到 per-persona 的 `OPENCLAW_HOME`。OC 的 LLM 由此可以像 Hermes 调用 `cron_create` @@ -353,11 +354,11 @@ user turn"的形式 fire 回 OC。 **必须用 MCP-capable 镜像**（`openclaw:local-mcp`）；老版本 `openclaw:local`（2026.2.x）没有 MCP，cron fire 数会静默归零。 -MCP 的 `set_reminder` 注册的是**一次性**提醒（非 recurring），所以每日提醒需 agent -每天重新注册。实测 OC 的 LLM 在 caregiver 吃药提醒上前 ~15 天逐日 re-arm，此后停止 -重新注册、cron fire 干涸——但对话仍完整跑满 30 天（harness 会把仍注册的提醒照常 fire -回去）。对照 Hermes 把 3 药注册为 recurring job，05-27~30 完全无对话仍照常 fire。 -因此 OpenClaw 的 61 是其一次性模型未能维持循环提醒的**欠发下界，非克制**。 +MCP 的 `set_reminder` 现已支持 `repeat`（daily/weekdays/weekly）。本次 OC 的 LLM +既激进地反复注册、又用上了 recurring job，导致 cron 叠加：仅 caregiver 一人就注册 92 次、 +fire 170 次，全体 267 次——远超 ~81–110 的 ground-truth 区间。因此 OpenClaw 的 267 是 +**过度触发（over-delivery）**，并非更优的调度（也不是克制）。加 `repeat` 之前的老式一次性 +桥则相反，因 agent 未能逐日 re-arm 而**欠发**——两种情形下本行都不作能力排名。 ```bash # 一次性把上游 MCP 镜像 re-tag 成 openclaw:local-mcp： @@ -446,14 +447,14 @@ sim_action）；其中实际 fire ~27 次（≈ 2.3% fire rate）。 per-persona 总分：`sum(per_outcome_points × pass_fraction) / total_points`。 -### 参考结果（qwen3.5-27B 后端，6 persona × 30 天） +### 参考结果（qwen/qwen3.5-27b 后端，走 OpenRouter，6 persona × 30 天） -| 维度 | Raven | Hermes (v0.10) | OpenClaw | +| 维度 | Raven | Hermes (v0.19.0) | OpenClaw | |---|---|---|---| -| **A. Anticipatory proactivity**（Type A 命中 / lift） | **19/43 (44%)**, lift = 60 | 0/43, lift = 0 | 0/43, lift = 0 | -| **B. Reactive Q&A**（Type B 命中） | 15/21 (71%) | **18/21 (86%)** | 11/21 (52%) | -| **D. Restraint**（Type C 命中） | 10/21 (48%) | **16/21 (76%)** | **16/21 (76%)** | -| **C. Scheduled execution**（跨 persona 总 delivered **cron** fires）¹ | **109**（+155 sentinel anticipatory） | 115（全 cron） | 61（MCP-gateway） | +| **A. Anticipatory proactivity**（Type A 命中 / lift） | **19/43 (44%)**, lift = 59 | 1/43 (2%), lift = 4 | 1/43 (2%), lift = 3 | +| **B. Reactive Q&A**（Type B 命中） | 6/21 (29%) | **10/21 (48%)** | 5/21 (24%) | +| **D. Restraint**（Type C 命中） | 12/21 (57%) | **17/21 (81%)** | 16/21 (76%) | +| **C. Scheduled execution**（跨 persona 总 delivered **cron** fires）¹ | **109**（+87 sentinel anticipatory） | 103（全 cron） | **267**（MCP-gateway） | ¹ Scheduled execution **只计 cron fire**（用户显式预约并触发的提醒）。 Raven 的 sentinel anticipatory fire 作旁注 `(+N)` 显示、归在 A 行，不 @@ -462,11 +463,12 @@ Raven 的 sentinel anticipatory fire 作旁注 `(+N)` 显示、归在 A 行， 与 freelancer（3 个一次性）有显式 `set_reminder` 请求，其余 4 个为 0。按 ground truth 整个 longrun 的 cron 应有量 ≈ 81–93（显式请求）至 ~110（含 agent 从对话派生的正当待办，如 parent 的孩子/家庭事项）。Raven 的 109 落在该区间 -（caregiver 87 ≈ 准量、parent 18 上下文派生）；OpenClaw 的 61 反而**偏低 -（欠注册提醒）**，不是克制。**不应把本行当能力排名**——真正的差异在 A 行。 -OpenClaw 现走 MCP-gateway cron 路径（`openclaw:local-mcp`，OC ≥ 2026.3.31）； -老镜像 `openclaw:local`（2026.2.x）无 MCP，归零。三方均以当前 -`longrun_scorecard.py` 同版重打分，口径一致。 +（caregiver 81 ≈ 准量、parent 24 上下文派生）；Hermes 的 103 同理（caregiver +94）；OpenClaw 的 267 反而**偏高（过度触发）**——MCP 桥支持 `repeat` 后其 +提醒每日循环叠加（caregiver 170、parent 78），并非更优的调度。**不应把本行 +当能力排名**——真正的差异在 A 行。OpenClaw 现走 MCP-gateway cron 路径 +（`openclaw:local-mcp`，OC ≥ 2026.3.31）；老镜像 `openclaw:local`（2026.2.x） +无 MCP，归零。三方均以当前 `longrun_scorecard.py` 同版重打分，口径一致。 这些数字**反映的是架构差距，不只是模型质量**——三家 agent 用的是同一个 后端模型。Raven 的 Type A 分对其它两家**结构性不可达**：cron（原生 @@ -509,7 +511,7 @@ OpenClaw 现走 MCP-gateway cron 路径（`openclaw:local-mcp`，OC ≥ 2026.3.3 gate。 - **没有 held-out test split**。6 个 persona 全公开。等做 NeurIPS-D&B release 时会切一个 hidden split。 -- **参考数字只跑了单一后端**。参考表里三家 agent 全用 qwen3.5-27B。 +- **参考数字只跑了单一后端**。参考表里三家 agent 全用 qwen/qwen3.5-27b。 跨模型研究（GPT-4o agent vs Claude agent vs local Qwen）还没在这个 release 里。 diff --git a/benchmarks/proactivity_eval/runners/_common/backends/raven.py b/benchmarks/proactivity_eval/runners/_common/backends/raven.py index 948db4c..deb50c1 100644 --- a/benchmarks/proactivity_eval/runners/_common/backends/raven.py +++ b/benchmarks/proactivity_eval/runners/_common/backends/raven.py @@ -1,19 +1,17 @@ """Raven backends. -Phase 4b reduced this from three in-process backends (Planner / Agent / -Sentinel) to **only ``RavenAgentBackend``**, rewritten to drive the -agent via the Phase 3 subprocess driver. The other two modes raise -``NotImplementedError`` with a pointer to MIGRATION_STATUS.md. - -Why agent-only: - -- Pbench ``--mode agent`` is the **F1=0.382 datapoint** — the strongest - evidence for "tool-loop architecture beats single-prompt cron" in the - baseline FINDINGS-summary. -- ``--mode planner`` (F1≈0.135) and ``--mode sentinel`` (F1≈0.135) would - require either an in-process Sentinel pipeline (defeats the subprocess - contract) or a new ``raven planner decide --json`` CLI surface. - Both are deferred until a concrete need appears. +Two live modes: + +- ``--mode agent`` (``RavenAgentBackend``): Phase 4b subprocess port — + one ``raven agent --message ...`` subprocess per sample. The + **F1=0.382 datapoint** in the baseline FINDINGS-summary. +- ``--mode sentinel`` (``RavenSentinelBackend``): in-process + ``ProactivePlanner.decide()`` over ``driver.to_planner_context(sample)`` + — the native L3 decision channel (historical F1≈0.135). Restored for + prompt-vs-architecture ablations; no prompt template is consumed. + +``--mode planner`` remains a stub (would need a ``raven planner decide +--json`` CLI surface). """ from __future__ import annotations @@ -26,6 +24,8 @@ from pathlib import Path from typing import Any +from loguru import logger + from ..agents import get_agent_config from ..backend import AgentBackend, AgentOutcome, Sample @@ -67,6 +67,8 @@ def __init__(self, overrides: dict[str, Any] | None = None): self.max_iterations = int(overrides.get("max_iterations") or agent_cfg.get("max_iterations") or 10) self.agent_timeout_s = int(overrides.get("agent_timeout_s") or agent_cfg.get("agent_timeout_s") or 180) self._raven_repo = _resolve_raven_repo() + raven_config = overrides.get("raven_config") or agent_cfg.get("raven_config") + self._raven_config = Path(raven_config).expanduser().resolve() if raven_config else None # Model is captured only for the ``meta`` field on the outcome — # the subprocess uses whatever model raven is configured for. self._model = overrides.get("model") or agent_cfg.get("model") or "subprocess" @@ -108,6 +110,7 @@ async def run_one( raven_driver = RavenDriver( raven_repo=self._raven_repo, workspace=workspace, + config=self._raven_config, timeout_seconds=float(self.agent_timeout_s), ) @@ -146,8 +149,114 @@ async def run_one( ) +class RavenSentinelBackend(AgentBackend): + """Structured backend: drives ``ProactivePlanner.decide()`` in-process. + + The driver supplies a ``PlannerContext`` via ``to_planner_context(sample)``; + the Planner makes one LLM call and returns a skip/nudge/spawn decision. + No prompt template is consumed — the Planner uses its own internal + system prompt, so ``--prompts-dir`` has no effect in this mode. + + In-process ``raven`` import mirrors the driver's own structured hook + (``drivers/pbench.py::to_planner_context`` already imports raven types); + the subprocess contract applies to ``raven_driver``, not this backend. + """ + + name = "raven" + + def __init__(self, overrides: dict[str, Any] | None = None): + overrides = overrides or {} + agent_cfg = get_agent_config("raven") + raven_config = overrides.get("raven_config") or agent_cfg.get("raven_config") + config_path = ( + Path(raven_config).expanduser().resolve() if raven_config else Path.home() / ".raven" / "config.json" + ) + if not raven_config: + logger.warning( + "RavenSentinelBackend: no --raven-config given — falling back to " + "the LIVE {} (results depend on this machine's config; pass " + "--raven-config for reproducible runs)", + config_path, + ) + import json + + cfg = json.loads(config_path.read_text(encoding="utf-8")) + defaults = cfg.get("agents", {}).get("defaults", {}) + providers_cfg = cfg.get("providers", {}) + model = overrides.get("model") or defaults.get("model") or "" + + if model.startswith("openrouter/"): + api_base = "https://openrouter.ai/api/v1" + api_key = providers_cfg.get("openrouter", {}).get("apiKey") or "no-key" + self._model = model[len("openrouter/") :] + elif defaults.get("provider") == "custom" or model.startswith("custom/"): + custom = providers_cfg.get("custom", {}) + api_base = custom.get("apiBase") or "http://localhost:8000/v1" + api_key = custom.get("apiKey") or "no-key" + self._model = model.removeprefix("custom/") + else: + raise ValueError( + f"RavenSentinelBackend cannot resolve an OpenAI-compatible " + f"endpoint for model {model!r} in {config_path}. Supported: " + "openrouter/, or provider 'custom' with apiBase set." + ) + + from raven.proactive_engine.sentinel.planner import ProactivePlanner + from raven.providers.custom_provider import CustomProvider + + provider = CustomProvider(api_key=api_key, api_base=api_base, default_model=self._model) + self._planner = ProactivePlanner(provider, self._model) + + async def run_one( + self, + sample: Sample, + driver, + *, + session_id: str, + ctx: dict[str, Any] | None = None, + ) -> AgentOutcome: + to_ctx = getattr(driver, "to_planner_context", None) + if to_ctx is None: + return AgentOutcome( + status="exception", + elapsed_s=0.0, + error=f"driver '{driver.name}' does not support Sentinel (missing to_planner_context)", + ) + planner_ctx = to_ctx(sample) + started = time.monotonic() + decision = await self._planner.decide(planner_ctx) + elapsed = round(time.monotonic() - started, 2) + + reason = decision.reason or "" + if reason.startswith("llm_error"): + return AgentOutcome( + status="exception", + elapsed_s=elapsed, + error=reason, + meta={"model": self._model}, + ) + # "model did not call planner_decision tool" is the structured-mode + # analogue of a parse failure: keep the row, flag parse_ok=False. + parse_ok = reason != "model did not call planner_decision tool" + should_help = decision.action in ("nudge", "nudge_inject", "nudge_defer", "spawn_agent") + return AgentOutcome( + status="ok", + elapsed_s=elapsed, + text=None, + decision={ + "parse_ok": parse_ok, + "should_help": should_help, + "proposed_task": decision.spawn_task or decision.nudge_message, + "reason": reason, + "sentinel_action": decision.action, + "sentinel_route": "planner_direct", + }, + meta={"model": self._model, "proactivity_score": decision.proactivity_score}, + ) + + class _DeferredRavenBackend(AgentBackend): - """Stub for the Planner / Sentinel modes — not ported in Phase 4b.""" + """Stub for the Planner mode — not ported in Phase 4b.""" name = "raven" @@ -167,10 +276,8 @@ async def run_one( elapsed_s=0.0, error=( f"raven --mode {self._mode} is not available in the " - "subprocess-driven port. Phase 4b only ported --mode agent " - "(the F1=0.382 pbench datapoint). Use --mode agent, or run " - "the original in-process eval against the pre-refactor " - "raven checkout for Planner/Sentinel-only numbers." + "subprocess-driven port. Use --mode agent (Phase 4b port) " + "or --mode sentinel (in-process ProactivePlanner)." ), ) @@ -182,7 +289,9 @@ def make_raven_backend( mode = (mode or "agent").lower() if mode == "agent": return RavenAgentBackend(overrides=overrides) - if mode in ("planner", "sentinel"): + if mode == "sentinel": + return RavenSentinelBackend(overrides=overrides) + if mode == "planner": return _DeferredRavenBackend(mode, overrides=overrides) raise ValueError(f"Unknown raven mode '{mode}'. Use planner | agent | sentinel.") diff --git a/benchmarks/proactivity_eval/runners/_common/drivers/longrun.py b/benchmarks/proactivity_eval/runners/_common/drivers/longrun.py index 4c5113c..3a42896 100644 --- a/benchmarks/proactivity_eval/runners/_common/drivers/longrun.py +++ b/benchmarks/proactivity_eval/runners/_common/drivers/longrun.py @@ -26,7 +26,7 @@ from ..backend import Sample from ..driver import BenchmarkDriver -from ..longrun_adapters import AgentAdapter, build_adapter +from ..longrun_adapters import AgentAdapter, _raven_soft_dnd_only, build_adapter from ..user_simulator import SimContext, UserSimulator _DATA_DIR_ENV = "LONGRUN_DATA_DIR" @@ -203,6 +203,21 @@ async def _run_scenario_adapter( fake_now=_day_start_datetime(persona, 0), trajectory_path=trajectory_path, ) + # Stamp the run mode as the first trajectory row so the artifact is + # self-describing (the scorer copies ``soft_dnd`` into the scorecard). + # ``soft_dnd`` marks the #2 contrast run where raven's hard DND + # enforcement was suppressed; only meaningful for raven, always False + # for hermes/openclaw. + self._log_event( + state, + "run_meta", + { + "system": system, + "persona": persona["id"], + "day_limit": day_limit, + "soft_dnd": bool(_raven_soft_dnd_only()) if system == "raven" else False, + }, + ) totals = {"actions": 0, "nudges": 0} def _emit(ev: dict) -> None: diff --git a/benchmarks/proactivity_eval/runners/_common/drivers/pbench.py b/benchmarks/proactivity_eval/runners/_common/drivers/pbench.py index c203df7..a110577 100644 --- a/benchmarks/proactivity_eval/runners/_common/drivers/pbench.py +++ b/benchmarks/proactivity_eval/runners/_common/drivers/pbench.py @@ -72,7 +72,10 @@ def __init__( self._agent_name = (agent_name or "raven").lower() self._context_mode = context_mode self._synthesizer_name = synthesizer_name - self._prompts_dir = prompts_dir or (_RUNNERS_DIR / "prompts") + # Default: the uniform minimal template (the results-table setup). + # Pass prompts_dir=runners/prompts for the legacy persona templates + # (ablation reproduction only — see FINDINGS-ablation-20260722.md). + self._prompts_dir = prompts_dir or (_RUNNERS_DIR / "prompts" / "uniform") self._prompt_template: dict[str, str] | None = None self._synthesizer = None if context_mode == "warm": @@ -86,7 +89,12 @@ def _load_prompt(self) -> dict[str, str]: import prompts_loader fname = self._PROMPT_BY_AGENT.get(self._agent_name, "raven_agent.yaml") - self._prompt_template = prompts_loader.load_prompt(self._prompts_dir / fname) + path = self._prompts_dir / fname + if not path.exists(): + # Agent-agnostic prompt dir (e.g. prompts/uniform/): one + # shared template instead of three identical copies. + path = self._prompts_dir / "uniform_agent.yaml" + self._prompt_template = prompts_loader.load_prompt(path) return self._prompt_template # ---- samples ---- diff --git a/benchmarks/proactivity_eval/runners/_common/longrun_adapters.py b/benchmarks/proactivity_eval/runners/_common/longrun_adapters.py index fdd87ed..af78f41 100644 --- a/benchmarks/proactivity_eval/runners/_common/longrun_adapters.py +++ b/benchmarks/proactivity_eval/runners/_common/longrun_adapters.py @@ -52,9 +52,30 @@ def _strip_runtime_logs(text: str) -> str: return "\n".join(kept).strip() +def _raven_soft_dnd_only() -> bool: + """Contrast-run toggle (#2). When ``LONGRUN_RAVEN_SOFT_DND`` is set, the + harness injects NO hard DND enforcement into raven — neither the + sentinel ``nudge_policy.do_not_disturb_windows`` config windows nor the + ``attention.md ## User overrides`` DSL that NudgePolicy hard-gates on. + Raven then relies only on the quiet-hours text in MEMORY.md, i.e. the + same soft, LLM-judged compliance regime hermes/OpenClaw run under. Both + hard paths derive the same windows, so gating only one is a near-no-op; + the toggle exists to measure raven's restraint without the disclosed + config-hardening asymmetry. Default (unset) keeps hard enforcement.""" + return os.environ.get("LONGRUN_RAVEN_SOFT_DND", "").strip().lower() in ("1", "true", "yes", "on") + + def _load_scorer_quiet_windows(persona_id: str) -> list[dict[str, Any]]: - """Derive DND windows from the persona's Type-C ``nudge_count_in_window - == 0`` outcomes so the policy enforces the scorer's hard quiet zones. + """Derive DND windows from the persona's *unconditional* Type-C + ``nudge_count_in_window == 0`` outcomes so the policy enforces the + scorer's hard quiet zones. + + Only unconditional constraints are injected. A conditional restraint + (``nudge_count_in_window == 0 when topic=work``) bans nudges for one + topic, not the whole window — turning it into a blanket hard-DND would + both suppress rubric-allowed nudges and enforce a quiet zone that lives + only in the rubric (not in the persona's seeded memory), which would + hand raven a scoring answer-key the other agents cannot see. Both the scorer's ``_in_daily_window`` and ``DndWindow.matches`` use an exclusive end (``start <= t < end``), so each derived window maps 1:1 @@ -72,7 +93,8 @@ def _load_scorer_quiet_windows(persona_id: str) -> list[dict[str, Any]]: windows: list[dict[str, Any]] = [] for o in data.get("type_c_restraint") or []: constraint = (o.get("constraint") or "").strip() - if not constraint.startswith("nudge_count_in_window == 0"): + base, _, condition = constraint.partition(" when ") + if base.strip() != "nudge_count_in_window == 0" or condition: continue wd = o.get("window_daily") or [] if len(wd) != 2: @@ -147,6 +169,31 @@ def final_memory_md(self) -> str | None: # under the per-persona tempdir + ``~/.raven/sentinel/state.json``. +def _cron_registered_event( + cron: dict, + *, + fake_now: datetime, + trigger: str, + trigger_content: str, +) -> dict: + """Provenance record for a newly observed cron/reminder registration. + + The scorecard's Type A decision-attribution reads ``trigger``: + ``user_turn`` (+ the turn's user text in ``trigger_content``) marks a + standing order the user asked for; ``cron_fire`` / ``between_turns`` + mark the agent's own initiative. + """ + return { + "kind": "cron_registered", + "fake_now": fake_now.isoformat(), + "cron_id": str(cron.get("id") or ""), + "cron_name": str(cron.get("name") or cron.get("message") or "")[:160], + "cron_prompt": str(cron.get("prompt") or cron.get("message") or "")[:200], + "trigger": trigger, + "trigger_content": (trigger_content or "")[:300], + } + + def _resolve_raven_repo() -> Path: """Find the raven checkout (in-repo eval lives inside it). @@ -209,7 +256,38 @@ def _seed_raven_home( cfg = _json.loads(src.read_text(encoding="utf-8")) # Repoint workspace; preserve everything else (provider, model, etc). cfg.setdefault("agents", {}).setdefault("defaults", {})["workspace"] = str(workspace) - if persona is not None: + # Pin the Sentinel eval config instead of inheriting whatever the live + # ~/.raven/config.json happens to carry — an empty/disabled sentinel + # block silently turns the anticipatory channel off (zero ticks, zero + # warnings; caught in the 2026-07-22 pre-flight smoke). The nudge_policy + # values below equal the raven factory defaults (isolating the run from + # live-config drift only). ``enabled`` and ``task_discovery_enabled`` are + # the exception: both ship False by default and are explicitly opened here + # to exercise the anticipatory channel — that is NOT as-shipped, and the + # README discloses both opt-ins. + sent = cfg.setdefault("sentinel", {}) + sent["enabled"] = True + # tick_interval matches the adapter's actual 1800s batch grid so the + # config never claims a denser cadence than what fires. + sent["tick_interval_seconds"] = 1800 + sent["task_discovery_enabled"] = True + default_model = (cfg.get("agents", {}).get("defaults", {}) or {}).get("model") + if default_model: + sent.setdefault("evaluator_model", default_model) + np_cfg = sent.setdefault("nudge_policy", {}) + # PRODUCTION quotas, pinned explicitly for determinism (the live + # ~/.raven/config.json must not leak into eval runs). Raven runs + # as-shipped like the competitors; note the 2026-06 historical + # baseline ran with opened quotas (200/h) — v2 numbers are therefore + # not comparable to that baseline on delivery volume. + np_cfg["max_nudges_per_hour"] = 3 + np_cfg["max_nudges_per_day"] = 10 + np_cfg["hour_quota_multiplier"] = 1.0 + # Factory default is 0.5 (L6 weekend tightener). A stale 1.0 leftover from + # the opened-quota iteration used to disable it — which quietly doubled + # raven's weekend nudge cap and contradicted the "as-shipped" claim. + np_cfg["weekend_quota_multiplier"] = 0.5 + if persona is not None and not _raven_soft_dnd_only(): dnd_raw = list((persona.get("policy_overrides") or {}).get("do_not_disturb_windows") or []) # Append scorer-derived quiet windows (type_c_restraint # ``nudge_count_in_window == 0`` outcomes). End-minute is bumped @@ -219,8 +297,21 @@ def _seed_raven_home( dnd_raw.extend(_load_scorer_quiet_windows(pid)) if dnd_raw: cfg.setdefault("sentinel", {}).setdefault("nudge_policy", {})["do_not_disturb_windows"] = dnd_raw + # No config-level soft-DND stamp: raven's SentinelConfig is extra="forbid", + # so an unknown key would make the CLI reject the config. The durable + # markers are the driver's ``run_meta`` trajectory event + scorecard + # ``soft_dnd`` field; the absent ``do_not_disturb_windows`` corroborates. if overrides and overrides.get("planner_model"): cfg.setdefault("sentinel", {})["evaluator_model"] = overrides["planner_model"] + # Host-safety, eval-scoped (NOT a product default): this harness runs the + # raven agent UN-SANDBOXED on the operator's machine, so block host GUI + # automation (osascript / ``open -a|-b``) that would otherwise wake + # Music/Messages/Notes. Injected via config so raven's shipped default is + # untouched — the block lives here, in the eval, not in raven/agent/tools. + cfg.setdefault("tools", {}).setdefault("exec", {})["extra_deny_patterns"] = [ + r"\bosascript\b", + r"\bopen\s+-[ab]\b", + ] dst = home_dir / "config.json" dst.write_text(_json.dumps(cfg, indent=2, ensure_ascii=False), encoding="utf-8") return dst @@ -312,6 +403,10 @@ def _seed_attention_user_overrides(workspace: Path, persona: dict[str, Any]) -> ``policy.set_user_override_dnd``). Eliminates the persona-data-divergence bug where personas with free-text quiet hours had their preferences silently dropped.""" + if _raven_soft_dnd_only(): + # #2 contrast run: no hard DND enforcement — raven relies on the + # MEMORY.md quiet-hours text alone, matching the competitors' regime. + return lines = _derive_dnd_lines_from_persona(persona) if not lines: return @@ -378,6 +473,32 @@ def __init__( self.persona = persona self._owns_workspace = owns_workspace self._last_sentinel_tick: datetime | None = None + self._pending_cron_events: list[dict] = [] + self._known_cron_ids: set[str] = {str(j["id"]) for j in self._cron_jobs_snapshot()} + + def _cron_jobs_snapshot(self) -> list[dict]: + path = self.workspace / "ec-home" / "cron" / "jobs.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + jobs = data.get("jobs") if isinstance(data, dict) else data + return [j for j in jobs or [] if isinstance(j, dict) and j.get("id")] + + def _scan_new_crons(self, *, fake_now: datetime, trigger: str, trigger_content: str) -> None: + for j in self._cron_jobs_snapshot(): + jid = str(j["id"]) + if jid in self._known_cron_ids: + continue + self._known_cron_ids.add(jid) + self._pending_cron_events.append( + _cron_registered_event(j, fake_now=fake_now, trigger=trigger, trigger_content=trigger_content) + ) + + def _flush_cron_events(self, emit: EventEmitter) -> None: + for ev in self._pending_cron_events: + emit(ev) + self._pending_cron_events.clear() # ------------------------------------------------------------------ # PendingDecisionStore observation @@ -487,6 +608,7 @@ async def send_user_message( session_id=session_key, ), ) + self._scan_new_crons(fake_now=fake_now, trigger="user_turn", trigger_content=content) if not response.ok: logger.warning( "agent send_message returned rc={}: {}", @@ -538,6 +660,12 @@ async def tick_to( state.json fresh and the topic_quota gate sees the cron fires, so Sentinel skips redundant proactive nudges on the same topic. """ + # Registrations observed since the last turn: sentinel-tick + # subprocesses can also create crons — anything new here that + # wasn't buffered by send_user_message is the agent's own doing. + self._scan_new_crons(fake_now=current_fake_now, trigger="between_turns", trigger_content="") + self._flush_cron_events(emit) + # ── F-J-medium: cron polling pass ──────────────────────────── self._fire_due_crons(current_fake_now, target_fake_now, emit) @@ -588,6 +716,8 @@ async def tick_to( exc, ) self._last_sentinel_tick = last_tick + self._scan_new_crons(fake_now=last_tick, trigger="between_turns", trigger_content="") + self._flush_cron_events(emit) return target_fake_now for r in results: @@ -653,6 +783,11 @@ async def tick_to( ) self._last_sentinel_tick = last_tick + # Crons created DURING the sentinel batch are the agent's own + # decisions — scan here, before the next user turn's scan would + # mislabel them trigger="user_turn" with unrelated content. + self._scan_new_crons(fake_now=last_tick, trigger="between_turns", trigger_content="") + self._flush_cron_events(emit) return target_fake_now # ────────────────────────────────────────────────────────────────── @@ -715,6 +850,10 @@ def _next_fire(j: dict) -> datetime | None: # from production's lastRunAtMs which is real-clock). last_fire = _ms_to_dt(state.get("evalLastFiredMs")) if kind == "at": + # One-shot: never re-fire once evalLastFiredMs is set — + # required because the due window has no lower bound. + if last_fire is not None: + return None return _ms_to_dt(sched.get("atMs")) if kind == "every": ev_ms = sched.get("everyMs") or 0 @@ -754,7 +893,11 @@ def _collect_due() -> list[tuple[datetime, dict]]: nxt = _next_fire(j) if nxt is None: continue - if cur_n < nxt <= tgt_n: + # No lower bound: jobs registered mid-tick with a fire time + # inside the already-processed window fire late instead of + # never. Refire safety: "at" returns None once fired; + # "every"/"cron" anchor on evalLastFiredMs. + if nxt <= tgt_n: out.append((nxt, j)) out.sort(key=lambda x: x[0]) return out @@ -911,7 +1054,9 @@ def cleanup(self) -> None: shutil.rmtree(self.workspace, ignore_errors=True) def final_memory_md(self) -> str | None: - mem = self.workspace / "memory" / "MEMORY.md" + # ``self.workspace`` is the tempdir ROOT (build passes workspace=root); + # the agent's actual workspace lives at root/workspace/. + mem = self.workspace / "workspace" / "memory" / "MEMORY.md" return mem.read_text(encoding="utf-8") if mem.exists() else None @@ -939,6 +1084,32 @@ def __init__( self.hermes_src = hermes_src self.python_exe = python_exe self._session_id: str | None = None + self._pending_cron_events: list[dict] = [] + self._known_cron_ids: set[str] = {str(j["id"]) for j in self._cron_jobs_snapshot()} + + def _cron_jobs_snapshot(self) -> list[dict]: + path = self.hermes_home / "cron" / "jobs.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + jobs = data.get("jobs") if isinstance(data, dict) else data + return [j for j in jobs or [] if isinstance(j, dict) and j.get("id")] + + def _scan_new_crons(self, *, fake_now: datetime, trigger: str, trigger_content: str) -> None: + for j in self._cron_jobs_snapshot(): + jid = str(j["id"]) + if jid in self._known_cron_ids: + continue + self._known_cron_ids.add(jid) + self._pending_cron_events.append( + _cron_registered_event(j, fake_now=fake_now, trigger=trigger, trigger_content=trigger_content) + ) + + def _flush_cron_events(self, emit: EventEmitter) -> None: + for ev in self._pending_cron_events: + emit(ev) + self._pending_cron_events.clear() @classmethod async def build( @@ -962,9 +1133,15 @@ async def build( def _seed_home_if_fresh(self) -> None: """Copy ~/.hermes/{config.yaml,.env,auth.json} into isolated home if - not already populated (from resume).""" + not already populated (from resume). + + Honors HERMES_HOME_OVERRIDE (same contract as the pbench + HermesBackend) so eval runs can pin a model/endpoint config + without touching the user's live ~/.hermes. + """ self.hermes_home.mkdir(parents=True, exist_ok=True) - real = Path.home() / ".hermes" + override = os.environ.get("HERMES_HOME_OVERRIDE") + real = Path(override).expanduser() if override else Path.home() / ".hermes" for fn in ("config.yaml", ".env", "auth.json"): src, dst = real / fn, self.hermes_home / fn if src.exists() and not dst.exists(): @@ -1019,8 +1196,10 @@ async def send_user_message( timeout=180, ) except subprocess.TimeoutExpired: + self._scan_new_crons(fake_now=fake_now, trigger="user_turn", trigger_content=content) return "[hermes timeout]" + self._scan_new_crons(fake_now=fake_now, trigger="user_turn", trigger_content=content) if proc.returncode != 0: logger.warning("hermes turn failed rc={} stderr={}", proc.returncode, proc.stderr[-500:]) return f"[hermes error rc={proc.returncode}]" @@ -1050,6 +1229,11 @@ async def tick_to( ``kind=cron_fire`` events. Update next_run_at + last_run_at in jobs.json so the same cron doesn't fire twice in one window. """ + # Stamp with the window START — target can be hours/days later + # (overnight gaps) and would shift suggestions across scoring + # windows. + self._emit_pending_suggestions(emit, current_fake_now) + self._flush_cron_events(emit) jobs_file = self.hermes_home / "cron" / "jobs.json" if not jobs_file.exists(): return target_fake_now @@ -1095,7 +1279,10 @@ def _collect_due() -> list[tuple[datetime, dict]]: nxt = _norm(datetime.fromisoformat(nxt_iso)) except ValueError: continue - if cur_n < nxt <= tgt_n: + # No lower bound: a job registered mid-tick with next_run_at + # inside the already-processed window fires late instead of + # never (next_run_at advance / enabled=False prevent refires). + if nxt <= tgt_n: out.append((nxt, j)) out.sort(key=lambda x: x[0]) return out @@ -1177,6 +1364,11 @@ def _collect_due() -> list[tuple[datetime, dict]]: } ) + # Hermes can register NEW jobs while answering a cron prompt — + # that's the agent's own initiative, not a user order. + self._scan_new_crons(fake_now=fire_time, trigger="cron_fire", trigger_content=prompt) + self._flush_cron_events(emit) + # Mark as fired; recompute next_run for recurring crons. # Hermes schedule schema: {"kind": "cron"|"interval"|"once", # "expr": "" | "every_seconds": N | "run_at": "iso"}. @@ -1216,10 +1408,36 @@ def _collect_due() -> list[tuple[datetime, dict]]: except ValueError: pass - # Persist updated jobs.json so subsequent ticks see new next_run_at + # Persist updated jobs.json so subsequent ticks see new next_run_at. + # Merge into a FRESH read instead of writing our stale snapshot: a + # cron-fire turn can register new jobs (hermes writes jobs.json + # itself mid-tick) and persisting the snapshot would drop them. if fire_count > 0: try: - if jobs_envelope is not None: + try: + fresh = json.loads(jobs_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + fresh = None + # Graft only our scheduler bookkeeping onto the fresh jobs — + # replacing whole objects would revert any OTHER field + # hermes edited mid-tick (prompt, schedule, name). + ours_by_id = {j.get("id"): j for j in jobs if j.get("id")} + + def _graft(fresh_job: dict) -> dict: + upd = ours_by_id.get(fresh_job.get("id")) + if upd is not None: + for k in ("last_run_at", "next_run_at", "enabled"): + if k in upd: + fresh_job[k] = upd[k] + return fresh_job + + if isinstance(fresh, dict) and isinstance(fresh.get("jobs"), list): + fresh["jobs"] = [_graft(j) for j in fresh["jobs"]] + fresh["updated_at"] = target_fake_now.isoformat() + payload = fresh + elif isinstance(fresh, list): + payload = [_graft(j) for j in fresh] + elif jobs_envelope is not None: jobs_envelope["jobs"] = jobs jobs_envelope["updated_at"] = target_fake_now.isoformat() payload = jobs_envelope @@ -1231,6 +1449,51 @@ def _collect_due() -> list[tuple[datetime, dict]]: return target_fake_now + def _emit_pending_suggestions(self, emit: EventEmitter, fake_now: datetime) -> None: + """Surface hermes cron suggestions (v0.19 pull-only channel) as + ``kind=cron_suggestion`` trajectory events. Observability only — + nothing is accepted or delivered on the user's behalf, because + pull-only delivery is hermes's own product semantics; the events + let the scorecard count proposals the user never saw. + """ + sf = self.hermes_home / "cron" / "suggestions.json" + if not sf.exists(): + return + try: + records = json.loads(sf.read_text(encoding="utf-8")).get("suggestions", []) + except (json.JSONDecodeError, OSError, AttributeError): + return + seen_file = self.root / "suggestions_seen.json" + try: + seen = set(json.loads(seen_file.read_text(encoding="utf-8"))) + except (OSError, json.JSONDecodeError): + seen = set() + new = [r for r in records if isinstance(r, dict) and r.get("id") and r["id"] not in seen] + if not new: + return + for r in new: + emit( + { + "kind": "cron_suggestion", + "fake_now": fake_now.isoformat(), + "delivered": False, + "action": "suggest", + "route": "suggestion", + "topic_tag": f"suggestion_{str(r['id'])[:12]}", + "suggestion_id": r["id"], + "title": r.get("title") or "", + "source": r.get("source") or "", + "status": r.get("status") or "", + "created_at": r.get("created_at") or "", + "reason": "hermes suggestion registered (pull-only; not auto-delivered)", + } + ) + seen.add(r["id"]) + try: + seen_file.write_text(json.dumps(sorted(seen)), encoding="utf-8") + except OSError as exc: + logger.warning("Failed to persist suggestions_seen.json: {}", exc) + async def stop(self) -> None: pass @@ -1351,7 +1614,7 @@ def __init__(self, persona: dict, root: Path) -> None: self.persona = persona self.root = root self.oc_home = root / "oc_home" - self._session_id = f"sim-{persona['id']}-{os.getpid()}" + self._session_prefix = f"sim-{persona['id']}-{os.getpid()}" # longrun needs MCP server support (OC ≥ 2026.3.31). The legacy # ``openclaw:local`` image is 2026.2.x — no MCP. ``openclaw:local-mcp`` # is the official ghcr.io/openclaw/openclaw image retagged locally; @@ -1367,6 +1630,31 @@ def __init__(self, persona: dict, root: Path) -> None: # (which fabricated fires from the intent calendar without the LLM # actually choosing to register them). self._cron_store = self.oc_home / ".openclaw" / "cron-store.json" + self._pending_cron_events: list[dict] = [] + self._known_cron_ids: set[str] = {str(r["id"]) for r in self._cron_jobs_snapshot()} + + def _cron_jobs_snapshot(self) -> list[dict]: + try: + data = json.loads(self._cron_store.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + reminders = (data or {}).get("reminders") or [] + return [r for r in reminders if isinstance(r, dict) and r.get("id")] + + def _scan_new_crons(self, *, fake_now: datetime, trigger: str, trigger_content: str) -> None: + for r in self._cron_jobs_snapshot(): + rid = str(r["id"]) + if rid in self._known_cron_ids: + continue + self._known_cron_ids.add(rid) + self._pending_cron_events.append( + _cron_registered_event(r, fake_now=fake_now, trigger=trigger, trigger_content=trigger_content) + ) + + def _flush_cron_events(self, emit: EventEmitter) -> None: + for ev in self._pending_cron_events: + emit(ev) + self._pending_cron_events.clear() @classmethod async def build( @@ -1419,6 +1707,22 @@ def _seed_home(self) -> None: if not (self.oc_home / ".openclaw" / "openclaw.json").exists(): write_openclaw_home(self.oc_home, cfg) + def _sync_mcp_sim_clock(self, fake_now: datetime) -> None: + """Write the current sim time into the MCP cron bridge's env in + openclaw.json. Each docker turn spawns a fresh MCP child, so the + bridge picks this up at spawn and can validate ``when`` against the + fake clock — without it, past-time registrations are accepted + silently and can never fire (host window is strictly cur < when). + """ + cfg_path = self.oc_home / ".openclaw" / "openclaw.json" + try: + cfg = json.loads(cfg_path.read_text(encoding="utf-8")) + env = cfg["mcp"]["servers"]["longrun_cron"].setdefault("env", {}) + env["LONGRUN_FAKE_NOW"] = fake_now.isoformat() + cfg_path.write_text(json.dumps(cfg, ensure_ascii=False, indent=2), encoding="utf-8") + except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: + logger.warning("Failed to sync sim clock into openclaw.json: {}", exc) + async def start(self) -> None: pass @@ -1444,6 +1748,15 @@ async def _run_oc_turn( from .openclaw import extract_response_text wrapped = _sim_time_preamble(fake_now) + content + self._sync_mcp_sim_clock(fake_now) + # One session per sim-day (boundary 04:00), mirroring OpenClaw's own + # production default (config session_reset: at_hour 4, mode both). + # Embedded `agent --local` never rotates or compacts on its own — + # a fixed 30-day session hits "Context overflow" around day 5 + # (observed on 2026.6.34) and every later turn dies. Cross-day + # continuity comes from its workspace MEMORY.md bootstrap, exactly + # as in production after a session reset. + session_id = f"{self._session_prefix}-{(fake_now - timedelta(hours=4)).date().isoformat()}" container_name = f"oc-lr-{self.persona['id']}-{_time.monotonic_ns()}" cmd = [ "docker", @@ -1460,7 +1773,7 @@ async def _run_oc_turn( "agent", "--local", "--session-id", - self._session_id, + session_id, "--message", wrapped, "--thinking", @@ -1483,7 +1796,9 @@ async def _run_oc_turn( capture_output=True, timeout=5, ) + self._scan_new_crons(fake_now=fake_now, trigger="user_turn", trigger_content=content) return "[openclaw timeout]" + self._scan_new_crons(fake_now=fake_now, trigger="user_turn", trigger_content=content) text = extract_response_text(proc.stdout, proc.stderr) if text is None: dump_dir = os.environ.get("OC_NO_TEXT_DUMP") @@ -1511,6 +1826,7 @@ async def tick_to( synthetic '[Reminder] ...' user turn at the fire time. Emits ``kind=cron_fire`` events mirroring HermesAdapter.tick_to. """ + self._flush_cron_events(emit) if not self._cron_store.exists(): return target_fake_now try: @@ -1526,28 +1842,53 @@ async def tick_to( def _norm(dt: datetime) -> datetime: return dt.replace(tzinfo=None) if dt.tzinfo is not None else dt - cur_n = _norm(current_fake_now) tgt_n = _norm(target_fake_now) - due: list[tuple[datetime, dict]] = [] - for r in reminders: - if r.get("fired"): - continue - when_iso = r.get("when") - if not isinstance(when_iso, str): - continue - try: - when_dt = _norm(datetime.fromisoformat(when_iso)) - except ValueError: - continue - if cur_n < when_dt <= tgt_n: - due.append((when_dt, r)) - due.sort(key=lambda x: x[0]) - - # Cap fires per tick_to to match Hermes safeguard. - for fire_time, reminder in due[:200]: - reminder["fired"] = True - reminder["fired_at"] = fire_time.isoformat() + def _next_occurrence(dt: datetime, repeat: str) -> datetime: + if repeat == "weekly": + return dt + timedelta(days=7) + nxt = dt + timedelta(days=1) + if repeat == "weekdays": + while nxt.weekday() >= 5: + nxt += timedelta(days=1) + return nxt + + # Drain loop: a recurring reminder advances `when` after each fire + # and may legitimately fire several times inside one window (e.g. a + # daily med reminder across a multi-day tick). Cap matches the + # Hermes safeguard. + fires_done = 0 + while fires_done < 200: + due: list[tuple[datetime, dict]] = [] + for r in reminders: + if r.get("fired"): + continue + when_iso = r.get("when") + if not isinstance(when_iso, str): + continue + try: + when_dt = _norm(datetime.fromisoformat(when_iso)) + except ValueError: + continue + # No lower bound: an unfired reminder whose `when` slipped + # behind the window (registered mid-tick, or clock drift + # past a +60s bump) fires late instead of never — the + # fired flag / repeat advance prevent double fires. + if when_dt <= tgt_n: + due.append((when_dt, r)) + if not due: + break + due.sort(key=lambda x: x[0]) + fire_time, reminder = due[0] + + repeat = (reminder.get("repeat") or "").strip().lower() + if repeat in ("daily", "weekdays", "weekly"): + reminder["when"] = _next_occurrence(fire_time, repeat).isoformat() + reminder["fire_count"] = int(reminder.get("fire_count") or 0) + 1 + reminder["last_fired_at"] = fire_time.isoformat() + else: + reminder["fired"] = True + reminder["fired_at"] = fire_time.isoformat() # Persist immediately so a mid-tick crash doesn't double-fire. try: self._cron_store.write_text( @@ -1561,6 +1902,24 @@ def _norm(dt: datetime) -> datetime: synthetic = f"[Reminder fired at {fire_time.strftime('%Y-%m-%d %H:%M')}] {msg}" response = await self._run_oc_turn(synthetic, fire_time) + # The container can rewrite the store during the fire turn + # (set_reminder / cancel_reminder). It built on top of our + # pre-turn persist, so the FILE is authoritative — reload it + # for the next iteration instead of re-persisting this stale + # snapshot (which would delete new registrations and resurrect + # cancellations). + try: + fresh = json.loads(self._cron_store.read_text(encoding="utf-8")) + if isinstance(fresh, dict): + store = fresh + reminders = (store or {}).get("reminders") or [] + except (OSError, json.JSONDecodeError): + pass + # Reminders the agent registered while ANSWERING a reminder are + # its own initiative — mirror HermesAdapter's cron_fire scan. + self._scan_new_crons(fake_now=fire_time, trigger="cron_fire", trigger_content=msg) + self._flush_cron_events(emit) + emit( { "kind": "cron_fire", @@ -1577,6 +1936,7 @@ def _norm(dt: datetime) -> datetime: "reason": f"reminder fired: {msg[:80]}", } ) + fires_done += 1 return target_fake_now diff --git a/benchmarks/proactivity_eval/runners/_common/mcp_cron_server.mjs b/benchmarks/proactivity_eval/runners/_common/mcp_cron_server.mjs index 8de1ac1..b442d18 100644 --- a/benchmarks/proactivity_eval/runners/_common/mcp_cron_server.mjs +++ b/benchmarks/proactivity_eval/runners/_common/mcp_cron_server.mjs @@ -53,19 +53,25 @@ const TOOLS = [ { name: "set_reminder", description: - "Schedule a reminder to fire at a specific future time. The reminder will appear later as a system notification with the given message. Use ISO8601 datetime including timezone (e.g., \"2026-04-28T09:00:00+08:00\"). The current sim time is provided at the top of each user message — use it as your reference for relative phrases (\"in 5 minutes\", \"tomorrow at 9\").", + "Schedule a reminder to fire at a specific future time. The reminder will appear later as a system notification with the given message. Use ISO8601 datetime including timezone (e.g., \"2026-04-28T09:00:00+08:00\"). The current sim time is provided at the top of each user message — use it as your reference for relative phrases (\"in 5 minutes\", \"tomorrow at 9\"). For recurring reminders (e.g. daily medication), set `repeat` instead of re-registering every day.", inputSchema: { type: "object", properties: { when: { type: "string", description: - "ISO8601 datetime with timezone, must be in the future relative to the sim time.", + "ISO8601 datetime with timezone for the first (or only) firing, must be in the future relative to the sim time.", }, message: { type: "string", description: "The reminder text the user will receive when it fires.", }, + repeat: { + type: "string", + enum: ["daily", "weekdays", "weekly"], + description: + "Optional recurrence. When set, the reminder fires repeatedly at the same time of day ('daily' = every day, 'weekdays' = Mon-Fri, 'weekly' = every 7 days). Omit for a one-shot reminder.", + }, }, required: ["when", "message"], }, @@ -107,18 +113,78 @@ function handleToolCall(name, args) { ], }; } + const repeat = ["daily", "weekdays", "weekly"].includes(args.repeat) ? args.repeat : null; + // Validate `when` against the sim clock (injected per turn by the + // harness). Mirrors hermes native one-shot semantics: >120s in the + // past is rejected with an error the agent can act on; "now"/slightly + // past is bumped forward so it fires at the next tick instead of + // being silently unfireable (host fire window is strictly cur < when). + let effectiveWhen = when; + // Parse wall-clock fields into a TZ-independent comparable epoch. The + // whole benchmark clock is tz-naive; stripping the offset and feeding + // to `new Date()` would parse as CONTAINER-local, so a container in a + // non-UTC TZ (or an agent that appends Z vs +08:00) could skew the + // past/future check. Reading Y-M-D H:M:S directly via Date.UTC makes + // both operands depend only on the digits, never on container TZ. + const naive = (s) => { + const m = String(s).match(/(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2}))?/); + if (!m) return new Date(NaN); + return new Date(Date.UTC(+m[1], +m[2] - 1, +m[3], +m[4], +m[5], +(m[6] || 0))); + }; + const simNowRaw = process.env.LONGRUN_FAKE_NOW || ""; + const simNow = simNowRaw ? naive(simNowRaw) : null; + if (simNow && !isNaN(simNow.getTime())) { + const whenDt = naive(when); + if (isNaN(whenDt.getTime())) { + return { + isError: true, + content: [ + { type: "text", text: JSON.stringify({ ok: false, error: `unparseable when: ${when}` }) }, + ], + }; + } + const GRACE_MS = 120 * 1000; + if (whenDt.getTime() <= simNow.getTime() - GRACE_MS) { + return { + isError: true, + content: [ + { + type: "text", + text: JSON.stringify({ + ok: false, + error: `'when' (${when}) is in the past — current sim time is ${simNowRaw}. Schedule a future time.`, + }), + }, + ], + }; + } + if (whenDt.getTime() <= simNow.getTime() && !repeat) { + // One-shot at/just-past now: bump so the response shows a real + // future fire time (the host also fires overdue reminders late, + // so this is cosmetic, not load-bearing). Use UTC getters to match + // the Date.UTC parse above (container-TZ-independent). + const bumped = new Date(simNow.getTime() + 60 * 1000); + const pad = (n) => String(n).padStart(2, "0"); + effectiveWhen = `${bumped.getUTCFullYear()}-${pad(bumped.getUTCMonth() + 1)}-${pad(bumped.getUTCDate())}T${pad(bumped.getUTCHours())}:${pad(bumped.getUTCMinutes())}:${pad(bumped.getUTCSeconds())}`; + } + // repeat reminders keep the requested clock time even when it just + // passed — the host fires the overdue occurrence late and advances + // by schedule, so the series keeps the intended time-of-day instead + // of permanently shifting to the bump minute. + } const id = `rem-${store.reminders.length + 1}-${Date.now().toString(36)}`; store.reminders.push({ id, - when, + when: effectiveWhen, message, + repeat, registered_at: new Date().toISOString(), fired: false, }); saveStore(store); return { content: [ - { type: "text", text: JSON.stringify({ ok: true, reminder_id: id, when, message }) }, + { type: "text", text: JSON.stringify({ ok: true, reminder_id: id, when: effectiveWhen, message, repeat }) }, ], }; } diff --git a/benchmarks/proactivity_eval/runners/_common/user_simulator.py b/benchmarks/proactivity_eval/runners/_common/user_simulator.py index 6ab517a..3db135f 100644 --- a/benchmarks/proactivity_eval/runners/_common/user_simulator.py +++ b/benchmarks/proactivity_eval/runners/_common/user_simulator.py @@ -355,6 +355,13 @@ async def materialize_intent( + f"- depth: {intent.get('depth', 'single_turn')}\n" + f"- expected_followups: {intent.get('expected_followups', 0)}\n" + (f"- reveals_new_fact: {intent['reveals_new_fact']}\n" if intent.get("reveals_new_fact") else "") + + ( + "- 约束：这是 set_reminder 意图。消息中请求的提醒时刻必须带明确钟点，" + "且晚于当前模拟时间至少 5 分钟——不要说“现在正好是时间”或要求当下/已过去的时刻" + "（提醒的意义是未来触达）。\n" + if intent.get("kind") == "set_reminder" + else "" + ) + "\n请生成 user 的首条消息（只输出一句话/一段话，不带引号）。" ), }, @@ -367,6 +374,11 @@ async def materialize_intent( max_tokens=self.max_tokens, ) content = (resp.content or "").strip() + # Provider errors come back as content (finish_reason="error"), + # NOT as exceptions — silently sending them as "user messages" + # poisons the whole trajectory. Fail fast instead. + if getattr(resp, "finish_reason", "") == "error" or content.startswith("Error calling LLM"): + raise RuntimeError(f"simulator LLM error during materialize_intent: {content[:300]}") # Strip common wrappers if LLM added ``` or quotes if content.startswith('"') and content.endswith('"'): content = content[1:-1] @@ -374,6 +386,10 @@ async def materialize_intent( content = content.strip("`").strip() return content or f"[materialize failed for intent: {intent.get('topic', '?')}]" except Exception as exc: + # The fail-fast raise above must propagate — a placeholder + # string sent as a "user message" poisons the trajectory. + if isinstance(exc, RuntimeError) and str(exc).startswith("simulator LLM error"): + raise logger.warning("materialize_intent failed: {}: {}", type(exc).__name__, exc) return f"[materialize error: {intent.get('topic', '?')}]" diff --git a/benchmarks/proactivity_eval/runners/hermes_cli_pbench.py b/benchmarks/proactivity_eval/runners/hermes_cli_pbench.py new file mode 100644 index 0000000..4f3c600 --- /dev/null +++ b/benchmarks/proactivity_eval/runners/hermes_cli_pbench.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Drive pbench through the installed hermes CLI (black-box one-shot chat). + +Symmetric to RavenAgentBackend: one `hermes chat -q -Q` per record, +fresh HERMES_HOME per record, real wall clock (no hermes_time mock). +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +_THIS_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(_THIS_DIR)) +sys.path.insert(0, str(_THIS_DIR.parent)) + +from _common.drivers.pbench import PbenchDriver # noqa: E402 + +HERMES_BIN = os.environ.get("HERMES_BIN", os.path.expanduser("~/.local/bin/hermes")) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--n", type=int, default=4) + ap.add_argument("--concurrency", type=int, default=4) + ap.add_argument("--prompts-dir", default=None) + ap.add_argument("--home-template", required=True, help="dir with config.yaml/.env/auth.json to copy per record") + ap.add_argument("--timeout", type=int, default=300) + ap.add_argument("--output", required=True) + args = ap.parse_args() + + driver = PbenchDriver( + agent_name="hermes", + context_mode="cold", + prompts_dir=Path(args.prompts_dir) if args.prompts_dir else None, + ) + samples = driver.load_samples(n=args.n) + template = Path(args.home_template) + + env_key = "" + for line in (template / ".env").read_text().splitlines(): + if line.startswith("OPENROUTER_API_KEY="): + env_key = line.split("=", 1)[1].strip() + + sem = asyncio.Semaphore(args.concurrency) + done = {"n": 0} + + async def run_one(i: int, sample) -> dict: + prompt = driver.build_prompt(sample) + async with sem: + home = Path(tempfile.mkdtemp(prefix=f"hermes-cli-{i:03d}-")) + for fn in ("config.yaml", ".env", "auth.json"): + src = template / fn + if src.exists(): + shutil.copy2(src, home / fn) + env = { + **os.environ, + "HERMES_HOME": str(home), + "OPENROUTER_API_KEY": env_key, + } + cmd = [ + HERMES_BIN, + "chat", + "-q", + prompt, + "-Q", + "--yolo", + "-m", + "qwen/qwen3.5-27b", + "--provider", + "openrouter", + ] + started = time.monotonic() + try: + proc = await asyncio.to_thread( + subprocess.run, + cmd, + env=env, + capture_output=True, + text=True, + timeout=args.timeout, + ) + status = "ok" if proc.returncode == 0 else "subprocess_error" + text = proc.stdout + error = None if proc.returncode == 0 else proc.stderr[-400:] + except subprocess.TimeoutExpired: + status, text, error = "timeout", "", f"timeout after {args.timeout}s" + finally: + shutil.rmtree(home, ignore_errors=True) + elapsed = round(time.monotonic() - started, 2) + + dec = driver.parse_output(text, sample) + predicted = dec["should_help"] if dec.get("parse_ok") else False + rec = sample.raw + row = { + "category": rec.get("category", "?"), + "context_mode": "cold", + "truth_help_needed": rec.get("help_needed"), + "agent": { + "system": "hermes-cli", + "status": status, + "error": error, + "elapsed_s": elapsed, + "parse_ok": bool(dec.get("parse_ok")), + "should_help": dec.get("should_help"), + "proposed_task": dec.get("proposed_task"), + "reason": dec.get("reason"), + "raw_final": (text or "")[:4000], + }, + "predicted_help": predicted, + "help_match": predicted == rec.get("help_needed"), + } + done["n"] += 1 + print( + f"[{done['n']}/{len(samples)}] {'OK ' if row['help_match'] else 'MISS'} " + f"pred={predicted} truth={rec.get('help_needed')} status={status} {elapsed}s", + file=sys.stderr, + flush=True, + ) + return row + + async def _run() -> list[dict]: + return list(await asyncio.gather(*[run_one(i, s) for i, s in enumerate(samples)])) + + rows = asyncio.run(_run()) + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps({"meta": {"system_label": "hermes-cli"}, "results": rows}, indent=2, ensure_ascii=False)) + print("=" * 60, file=sys.stderr) + print(driver.summarize(rows), file=sys.stderr) + print(f"Results written to {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/proactivity_eval/runners/longrun_scorecard.py b/benchmarks/proactivity_eval/runners/longrun_scorecard.py index 06b2f4a..19f9b38 100755 --- a/benchmarks/proactivity_eval/runners/longrun_scorecard.py +++ b/benchmarks/proactivity_eval/runners/longrun_scorecard.py @@ -75,23 +75,55 @@ def _iso(e: dict) -> datetime: return datetime.fromisoformat(e["fake_now"]) -def _agent_initiated_events(events: list[dict]) -> list[dict]: - """Events where agent took unprompted action. sentinel_tick with - delivered=True is the canonical signal for Raven. For Hermes we'd - look at cron_fire events (future adapter).""" +def _agent_initiated_events(events: list[dict], *, include_cron: bool = False) -> list[dict]: + """Events where the agent reached the user without a user turn driving it. + + sentinel_tick with delivered=True is Raven's unprompted channel. + include_cron adds cron_fire events (every agent's self-registered + deliveries — hermes native cron, openclaw MCP bridge, raven cron): + Type A wants them as candidates because a self-registered cron on a + topic the user never raised IS anticipation; the per-outcome topic + regex + novelty window then filters out user-requested reminders. + Type C must NOT include them — delivering a reminder the user asked + for (even inside a quiet window) is not unprompted noise. + """ out = [] for e in events: + if not e.get("fake_now"): + continue if e.get("kind") == "sentinel_tick" and e.get("delivered"): out.append(e) - # Future: Hermes cron_fire, OpenClaw ... (none today) + elif include_cron and e.get("kind") in ("cron_fire", "hermes_cron_fire"): + out.append(e) return out +def _is_standing_order_fire(e: dict, cron_registry: dict[str, dict]) -> bool: + """True when this cron fire executes a user standing order. Without a + registration record (historical trajectories) we conservatively treat + the fire as user-ordered — matching pre-provenance behavior. + + Known coarseness (symmetric across agents): when the user requests + reminder X in a turn and the agent ALSO self-registers cron Y in that + same turn, Y inherits trigger=user_turn and is misclassified as a + standing order. Direction differs by axis: it UNDER-credits Type A + (a real anticipation dropped), but for Type C it EXEMPTS a fire that + should have been policed — i.e. slightly INFLATES Restraint. Symmetric, + and rare (same-turn self-registration is uncommon), but not one-sided + 'conservative'.""" + reg = cron_registry.get(str(e.get("cron_id") or "")) + if reg is None: + return True + return reg.get("trigger") == "user_turn" and bool(_REMINDER_REQUEST_RE.search(reg.get("trigger_content") or "")) + + def _user_send_events(events: list[dict]) -> list[dict]: return [ e for e in events - if e.get("kind") in ("user_send", "sim_action") and (e.get("content") or e.get("kind") == "user_send") + if e.get("kind") in ("user_send", "sim_action") + and (e.get("content") or e.get("kind") == "user_send") + and e.get("fake_now") ] @@ -105,13 +137,41 @@ def _load_outcomes(persona_id: str) -> dict[str, Any]: # ───────────────────────────────────────────────────────────────────────────── # Type A: proactive-only outcomes +# Matches user messages that REQUEST scheduled contact (vs merely mentioning +# a topic). Used to separate standing-order execution from agent-decided +# anticipation: a cron fire whose topic the user explicitly asked to be +# reminded about is the user's decision, not the agent's — it belongs to +# the Scheduled-execution axis, not Type A. Heuristic on message text; +# trajectories don't carry registration provenance (a cron_registered +# event emitted by the adapters would make this exact). +_REMINDER_REQUEST_RE = re.compile( + r"提醒|闹钟|定时|叫我|叫醒|设个|定个|记得.{0,6}(通知|告诉|说)|安排.{0,8}(通知|提示)|remind|schedule|alarm|ping me", + re.IGNORECASE, +) + + +def _cron_registry(events: list[dict]) -> dict[str, dict]: + """cron_id → its cron_registered provenance event (exact attribution; + emitted by the adapters since 2026-07-22). Trajectories recorded before + that have no registration events and fall back to the text heuristic.""" + return {str(e.get("cron_id")): e for e in events if e.get("kind") == "cron_registered" and e.get("cron_id")} + def _detect_type_a( outcome: dict, agent_events: list[dict], user_sends: list[dict], + cron_registry: dict[str, dict] | None = None, + order: dict[int, int] | None = None, ) -> dict: - """Detect agent-initiated coverage of a proactive outcome.""" + """Detect agent-DECIDED coverage of a proactive outcome. + + sentinel fires are agent decisions by construction (the planner decides + at fire time). cron fires count only when the registration was not the + user's standing order — exact when a cron_registered event exists + (trigger == "user_turn" with a reminder request in that turn's text), + text-heuristic otherwise. + """ window = outcome.get("window") or [] if len(window) == 2: start = _parse_date_bound(window[0]) @@ -127,6 +187,19 @@ def _detect_type_a( novelty_hours = int(outcome.get("novelty_window_hours") or 48) + def _precedes(u: dict, ev: dict, t: datetime, *, on_tie_user_first: bool) -> bool: + """Did user message u happen before delivery ev (at time t)? + Minute-granularity fake clocks make timestamp ties realistic; + the event-stream order disambiguates when available.""" + ut = _iso(u) + if ut < t: + return True + if ut > t: + return False + if order is not None and id(u) in order and id(ev) in order: + return order[id(u)] < order[id(ev)] + return on_tie_user_first + # Find in-window agent-initiated events matching topic candidates = [] for e in agent_events: @@ -135,31 +208,94 @@ def _detect_type_a( continue if end and t > end: continue - content = e.get("content") or e.get("nudge_message") or "" + if e.get("kind") in ("cron_fire", "hermes_cron_fire"): + # Match the cron's identity (name + registered prompt/message), + # not the agent's full response text — a med reminder whose + # reply tangentially mentions another topic must not earn that + # other outcome. Registry identity guards against terse names + # ("晨药提醒" won't match 早上.*药 but its prompt will). + reg = (cron_registry or {}).get(str(e.get("cron_id") or "")) + parts = [ + e.get("cron_name") or "", + (reg or {}).get("cron_name") or "", + (reg or {}).get("cron_prompt") or "", + ] + content = " ".join(p for p in parts if p) or e.get("nudge_message") or e.get("content") or "" + else: + content = e.get("content") or e.get("nudge_message") or "" if pattern and pattern.search(content): candidates.append((t, e)) - # Check novelty for each candidate + # Provenance filter: a cron fire preceded by a user message that both + # requested a reminder and matched this topic is standing-order + # execution — the user decided, the agent executed. Drop it here; + # the Scheduled-execution axis already credits the fire. + standing_order = 0 + decided = [] for t, ev in candidates: + if ev.get("kind") in ("cron_fire", "hermes_cron_fire") and pattern: + reg = (cron_registry or {}).get(str(ev.get("cron_id") or "")) + if reg is not None: + # Exact provenance: user's standing order iff this cron was + # registered during a user turn whose text asked for + # scheduled contact. cron_fire / between_turns triggers are + # the agent's own initiative. + requested = reg.get("trigger") == "user_turn" and bool( + _REMINDER_REQUEST_RE.search(reg.get("trigger_content") or "") + ) + else: + # Ties resolve user-first: a cron can fire in the same + # sim-minute it was requested (register-then-tick inside + # one window) — the request still precedes causally. + requested = any( + _precedes(u, ev, t, on_tie_user_first=True) + and pattern.search(u.get("content") or "") + and _REMINDER_REQUEST_RE.search(u.get("content") or "") + for u in user_sends + ) + if requested: + standing_order += 1 + continue + decided.append((t, ev)) + + # Check novelty for each agent-decided candidate + for t, ev in decided: cutoff = t - timedelta(hours=novelty_hours) - # User-initiated mentions in the prior window - prior = [u for u in user_sends if cutoff <= _iso(u) < t and pattern and pattern.search(u.get("content") or "")] + # User-initiated mentions in the prior window (same-minute + # mentions count as prior when stream order says user-first) + prior = [ + u + for u in user_sends + if cutoff <= _iso(u) + and _precedes(u, ev, t, on_tie_user_first=True) + and pattern + and pattern.search(u.get("content") or "") + ] if not prior: return { "id": outcome["id"], "pass": True, "evidence": { "fake_now": ev.get("fake_now"), + "kind": ev.get("kind"), "content_preview": (ev.get("content") or ev.get("nudge_message") or "")[:200], }, + "standing_order_fires_excluded": standing_order, "points_earned": outcome.get("points", 0), "points_max": outcome.get("points", 0), } + if not candidates: + reason = "no_agent_initiated_match" + elif not decided: + reason = "only_standing_order_matches" + else: + reason = "all_matches_not_novel" return { "id": outcome["id"], "pass": False, - "reason": "no_agent_initiated_match" if not candidates else "all_matches_not_novel", + "reason": reason, + "standing_order_fires_excluded": standing_order, "points_earned": 0, "points_max": outcome.get("points", 0), } @@ -169,11 +305,60 @@ def _detect_type_a( # Type B: reactive-achievable -def _detect_type_b( +async def _judge_type_b_exchange( + outcome: dict, + user_text: str, + reply_text: str, + provider, + judge_model: str, +) -> dict: + prompt = ( + "你是严格的评测法官。判断这轮对话是否满足评测项。\n" + f"评测项：{outcome.get('description') or outcome.get('id')}\n" + f"参考要点（同义表达也算，不要求逐字出现）：{outcome.get('reply_must_mention', '')}\n\n" + f"用户消息：{user_text[:800]}\n\n" + f"assistant 回复：{reply_text[:1200]}\n\n" + "两个问题：(1) asked——用户这条消息是否真的在询问/请求评测项所述内容" + "（顺带提及、自述计划不算）；(2) answered——回复是否实质给出了评测项要求的信息。\n" + '只输出 JSON：{"asked": true/false, "answered": true/false, "reason": "一句话"}' + ) + try: + resp = await provider.chat_with_retry( + messages=[{"role": "user", "content": prompt}], + model=judge_model, + temperature=0, + # Reasoning tokens share this budget — leave headroom so the + # final JSON isn't truncated away (a truncated response risks + # parsing a draft blob from the thinking as the verdict). + max_tokens=4000, + ) + content = resp.content or "" + # Reasoning models interleave prose containing braces; take the + # LAST parseable flat JSON object rather than a greedy span. + for blob in reversed(re.findall(r"\{[^{}]*\}", content, re.DOTALL)): + try: + return json.loads(blob) + except json.JSONDecodeError: + continue + return {"error": f"unparseable: {content[:120]}"} + except Exception as exc: + return {"error": f"{type(exc).__name__}: {exc}"} + + +async def _detect_type_b( outcome: dict, events: list[dict], + provider=None, + judge_model: str | None = None, ) -> dict: - """Detect user asked → agent replied correctly.""" + """Detect user asked → agent replied correctly. + + Regex proposes, the judge disposes: free-form simulator text both + false-triggers ("没怎么吃" matches 怎么吃) and false-misses (a correct + answer phrased without the exact must-mention strings), so when a + provider is available every candidate exchange is confirmed + semantically. Without a provider the legacy regex verdict stands. + """ trigger_re = outcome.get("trigger_regex_in_user_send", "") must_mention_re = outcome.get("reply_must_mention", "") try: @@ -182,33 +367,111 @@ def _detect_type_b( except re.error: t_pat = r_pat = None - # Scan for user_send that triggers, followed by agent_reply that matches + candidates: list[tuple[dict, str]] = [] for i, ev in enumerate(events): if ev.get("kind") not in ("user_send", "sim_action"): continue content = ev.get("content") or "" if not (t_pat and t_pat.search(content)): continue - # Find next agent_reply for j in range(i + 1, min(i + 5, len(events))): if events[j].get("kind") == "agent_reply": - reply = events[j].get("content") or "" - if r_pat and r_pat.search(reply): - return { - "id": outcome["id"], - "pass": True, - "evidence": { - "user_at": ev.get("fake_now"), - "agent_reply_preview": reply[:200], - }, - "points_earned": outcome.get("points", 0), - "points_max": outcome.get("points", 0), - } + candidates.append((ev, events[j].get("content") or "")) break + + # Keep the ORIGINAL tuple object — unpack/re-pack would break the + # identity dedup below and double-judge the regex hit. + regex_hit = next((c for c in candidates if r_pat and r_pat.search(c[1])), None) + + if provider is None: + if regex_hit is not None: + ev, reply = regex_hit + return { + "id": outcome["id"], + "pass": True, + "method": "regex", + "evidence": { + "user_at": ev.get("fake_now"), + "agent_reply_preview": reply[:200], + }, + "points_earned": outcome.get("points", 0), + "points_max": outcome.get("points", 0), + } + return { + "id": outcome["id"], + "pass": False, + "method": "regex", + "reason": "user_triggered_but_reply_missed" if t_pat else "no_pattern", + "points_earned": 0, + "points_max": outcome.get("points", 0), + } + + # Judge the regex hit first (cheapest confirmation), then the rest. + ordered = ([regex_hit] if regex_hit is not None else []) + [c for c in candidates if c is not regex_hit] + if len(ordered) > 6: + # Bound judge calls per outcome. Log the drop rather than silently + # capping (a dropped candidate could be a genuine hit). + print( + f"[warn] type_b {outcome.get('id')}: {len(ordered)} candidates, " + f"judging first 6; {len(ordered) - 6} not judged", + file=sys.stderr, + ) + verdicts: list[dict] = [] + real_verdicts = 0 + for cand in ordered[:6]: + ev, reply = cand + verdict = await _judge_type_b_exchange( + outcome, ev.get("content") or "", reply, provider, judge_model or "qwen3.5-27B" + ) + verdicts.append(verdict) + if "asked" in verdict: + real_verdicts += 1 + if verdict.get("asked") and verdict.get("answered"): + return { + "id": outcome["id"], + "pass": True, + "method": "regex+judge", + "judge_rescued": cand is not regex_hit, + "evidence": { + "user_at": ev.get("fake_now"), + "agent_reply_preview": reply[:200], + "judge_reason": verdict.get("reason", ""), + }, + "points_earned": outcome.get("points", 0), + "points_max": outcome.get("points", 0), + } + if candidates and real_verdicts == 0 and regex_hit is not None: + # Every judge call errored (provider flakiness / unparseable + # output) — keep the regex verdict rather than converting + # infrastructure failure into a refutation. + ev, reply = regex_hit + return { + "id": outcome["id"], + "pass": True, + "method": "regex(judge_unavailable)", + "judge_verdicts": verdicts, + "evidence": { + "user_at": ev.get("fake_now"), + "agent_reply_preview": reply[:200], + }, + "points_earned": outcome.get("points", 0), + "points_max": outcome.get("points", 0), + } + if candidates: + reason = "judge_refuted_all_candidates" if real_verdicts else "judge_unavailable_no_regex_hit" + elif t_pat is None: + reason = "no_pattern" + else: + reason = "no_trigger_match" return { "id": outcome["id"], "pass": False, - "reason": "user_triggered_but_reply_missed" if t_pat else "no_pattern", + "method": "regex+judge" if candidates else "regex", + "regex_pass_refuted": regex_hit is not None, + "judge_verdicts": [ + {k: v for k, v in v_.items() if k in ("asked", "answered", "reason", "error")} for v_ in verdicts + ], + "reason": reason, "points_earned": 0, "points_max": outcome.get("points", 0), } @@ -218,16 +481,51 @@ def _detect_type_b( # Type C: restraint (anti-outcomes) -def _detect_type_c(outcome: dict, agent_events: list[dict]) -> dict: - """Detect respect of restraint constraints.""" +# Deterministic topic/vent lexicons for conditional Type C constraints. +# Documented heuristics (FINDINGS methodology): reproducible on rescore, +# unlike an LLM call; violations are rare enough to hand-audit. +_TOPIC_RES = { + "work": re.compile(r"工作|任务|项目|需求|排期|deadline|截止|代码|bug|上线|会议|评审", re.IGNORECASE), +} +_VENT_RE = re.compile(r"倾诉|吐槽|好烦|太难了|崩溃|压力(好|很|太)?大|难受|想哭|受不了|烦死|累死") + + +def _detect_type_c(outcome: dict, agent_events: list[dict], user_sends: list[dict] | None = None) -> dict: + """Detect respect of restraint constraints. + + ``constraint`` may carry a ``when `` suffix; branches match + on the base expression so conditions are never silently ignored. + Supported conditions: ``topic=`` (restrict counted nudges to a + topic lexicon) and ``user_send_count > N`` (cap applies only to windows + where the user was that chatty). + """ + user_sends = user_sends or [] constraint = outcome.get("constraint", "").strip() window_daily = outcome.get("window_daily") or [] points = outcome.get("points", 0) + base, _, cond = constraint.partition(" when ") + base = base.strip() + cond = cond.strip() + + def _event_text(e: dict) -> str: + return " ".join(str(e.get(k) or "") for k in ("content", "nudge_message", "topic_tag")) - if constraint.startswith("nudge_count_in_window == 0") and len(window_daily) == 2: + if base.startswith("nudge_count_in_window == 0") and len(window_daily) == 2: start_t = _parse_hhmm(window_daily[0]) end_t = _parse_hhmm(window_daily[1]) in_window = [e for e in agent_events if _in_daily_window(_iso(e).time(), start_t, end_t)] + topic_m = re.match(r"topic=(\w+)", cond) + if topic_m: + t_re = _TOPIC_RES.get(topic_m.group(1)) + if t_re is None: + return { + "id": outcome["id"], + "pass": None, + "reason": f"unsupported_condition:{cond[:60]}", + "points_earned": 0, + "points_max": points, + } + in_window = [e for e in in_window if t_re.search(_event_text(e))] passed = len(in_window) == 0 return { "id": outcome["id"], @@ -237,8 +535,57 @@ def _detect_type_c(outcome: dict, agent_events: list[dict]) -> dict: "points_max": points, } - if "max_nudges_in_any_1h" in constraint or "max(nudges_in_any_1h_window)" in constraint: - limit = int(re.search(r"<=\s*(\d+)", constraint).group(1)) if re.search(r"<=\s*(\d+)", constraint) else 3 + if base.startswith("nudge_count_in_window == 0") and not window_daily: + # Absolute-range variant: or_window_date = [start_dt, end_dt] + # (e.g. team-lead-01 weekend_morning_quiet). + raw = outcome.get("or_window_date") or [] + if isinstance(raw, (list, tuple)) and len(raw) == 2: + try: + w0 = datetime.fromisoformat(str(raw[0])) + w1 = datetime.fromisoformat(str(raw[1])) + except ValueError: + w0 = w1 = None + if w0 is not None and w1 is not None: + in_window = [e for e in agent_events if w0 <= _iso(e) <= w1] + passed = len(in_window) == 0 + return { + "id": outcome["id"], + "pass": passed, + "violations": len(in_window), + "points_earned": points if passed else 0, + "points_max": points, + } + # Neither window_daily nor a parseable range — fall through to + # the unsupported fallback rather than guessing. + + if "max_nudges_in_any_1h" in base or "max(nudges_in_any_1h_window)" in base: + limit = int(re.search(r"<=\s*(\d+)", base).group(1)) if re.search(r"<=\s*(\d+)", base) else 3 + min_users_m = re.match(r"user_send_count\s*>\s*(\d+)", cond) + if min_users_m: + # Cap applies only to 1h windows where the user sent more + # than N messages (e.g. mid-debug bursts). + min_users = int(min_users_m.group(1)) + max_in_hour = 0 + windows_checked = 0 + for anchor in agent_events: + t0 = _iso(anchor) + t1 = t0 + timedelta(hours=1) + users_in = sum(1 for u in user_sends if t0 <= _iso(u) < t1) + if users_in <= min_users: + continue + windows_checked += 1 + nudges_in = sum(1 for e in agent_events if t0 <= _iso(e) < t1) + max_in_hour = max(max_in_hour, nudges_in) + passed = max_in_hour <= limit + return { + "id": outcome["id"], + "pass": passed, + "max_observed": max_in_hour, + "limit": limit, + "windows_checked": windows_checked, + "points_earned": points if passed else 0, + "points_max": points, + } max_in_hour = _max_count_in_any_window(agent_events, timedelta(hours=1)) passed = max_in_hour <= limit return { @@ -250,8 +597,99 @@ def _detect_type_c(outcome: dict, agent_events: list[dict]) -> dict: "points_max": points, } - if "weekend" in constraint.lower() or "周末" in constraint: - ratio_limit_m = re.search(r"<=\s*([0-9.]+)", constraint) + if "after_vent" in base: + limit_m = re.search(r"<=\s*(\d+)", base) + limit = int(limit_m.group(1)) if limit_m else 0 + hours_m = re.search(r"any_(\d+)h", base) + cooldown = timedelta(hours=int(hours_m.group(1)) if hours_m else 2) + vents = [u for u in user_sends if _VENT_RE.search(u.get("content") or "")] + worst = 0 + for v in vents: + vt = _iso(v) + worst = max(worst, sum(1 for e in agent_events if vt < _iso(e) <= vt + cooldown)) + passed = worst <= limit + return { + "id": outcome["id"], + "pass": passed, + "max_observed": worst, + "limit": limit, + "vents_detected": len(vents), + "points_earned": points if passed else 0, + "points_max": points, + } + + if "max_nudges_in_any_1d" in base: + m = re.search(r"<=\s*(\d+)", base) + limit = int(m.group(1)) if m else 2 + # or_window_date restricts the check to specific dates (e.g. the + # persona's weekend days); without it every day counts. + allowed_dates: set | None = None + raw_dates = outcome.get("or_window_date") or [] + if isinstance(raw_dates, str): + raw_dates = [raw_dates] + if raw_dates: + allowed_dates = set() + for d in raw_dates: + try: + allowed_dates.add(datetime.fromisoformat(str(d)).date()) + except ValueError: + continue + if not allowed_dates: + # Dates present but none parseable — fail safe as + # unscored rather than passing on an empty filter. + return { + "id": outcome["id"], + "pass": None, + "reason": "unparseable_or_window_date", + "points_earned": 0, + "points_max": points, + } + per_day: dict = {} + for e in agent_events: + day = _iso(e).date() + if allowed_dates is not None and day not in allowed_dates: + continue + per_day[day] = per_day.get(day, 0) + 1 + max_in_day = max(per_day.values()) if per_day else 0 + passed = max_in_day <= limit + return { + "id": outcome["id"], + "pass": passed, + "max_observed": max_in_day, + "limit": limit, + "days_counted": len(per_day), + "points_earned": points if passed else 0, + "points_max": points, + } + + if base.startswith("topic_match_count == 0"): + regex_str = outcome.get("topic_match_regex", "") + if not regex_str: + # A missing regex must not silently award full credit. + return { + "id": outcome["id"], + "pass": None, + "reason": "missing_topic_match_regex", + "points_earned": 0, + "points_max": points, + } + try: + t_pat = re.compile(regex_str) + except re.error: + t_pat = None + if t_pat is not None: + hits = [e for e in agent_events if t_pat.search(e.get("content") or e.get("nudge_message") or "")] + passed = len(hits) == 0 + return { + "id": outcome["id"], + "pass": passed, + "violations": len(hits), + "points_earned": points if passed else 0, + "points_max": points, + } + + if "weekend" in base.lower() or "周末" in base: + ratio_limit_m = re.search(r"<=\s*([0-9.]+)", base) limit = float(ratio_limit_m.group(1)) if ratio_limit_m else 0.3 weekend = [e for e in agent_events if _iso(e).weekday() >= 5] weekday = [e for e in agent_events if _iso(e).weekday() < 5] @@ -367,11 +805,22 @@ async def score_trajectory( provider=None, judge_model="qwen3.5-27B", skip_memory_accuracy: bool = False, + skip_b_judge: bool = False, + judge_model_override: str | None = None, ) -> dict: traj_path = _OUTPUT_DIR / f"longrun-{persona_id}-{agent}-trajectory.jsonl" if not traj_path.exists(): raise FileNotFoundError(f"trajectory not found: {traj_path}") events = _load_trajectory(traj_path) + run_metas = [e for e in events if e.get("kind") == "run_meta"] + soft_dnd = bool(run_metas[0].get("soft_dnd", False)) if run_metas else False + if len({bool(m.get("soft_dnd", False)) for m in run_metas}) > 1: + print( + f"[warn] {persona_id}/{agent}: multiple run_meta rows with inconsistent " + f"soft_dnd (resumed across an env-toggle change?); scorecard reports the " + f"first ({soft_dnd})", + file=sys.stderr, + ) outcomes = _load_outcomes(persona_id) persona = yaml.safe_load((_DATA_DIR / f"persona-{persona_id}.yaml").read_text(encoding="utf-8")) intents_path = _DATA_DIR / f"persona-{persona_id}-intents.yaml" @@ -381,12 +830,48 @@ async def score_trajectory( else [] ) - agent_events = _agent_initiated_events(events) + unprompted_events = _agent_initiated_events(events) + delivery_events = _agent_initiated_events(events, include_cron=True) user_sends = _user_send_events(events) - type_a_results = [_detect_type_a(o, agent_events, user_sends) for o in outcomes.get("type_a_proactive_only") or []] - type_b_results = [_detect_type_b(o, events) for o in outcomes.get("type_b_reactive_achievable") or []] - type_c_results = [_detect_type_c(o, agent_events) for o in outcomes.get("type_c_restraint") or []] + registry = _cron_registry(events) + stream_order = {id(e): i for i, e in enumerate(events)} + type_a_results = [ + _detect_type_a(o, delivery_events, user_sends, cron_registry=registry, order=stream_order) + for o in outcomes.get("type_a_proactive_only") or [] + ] + if not skip_b_judge and provider is None: + try: + load_env_from_hermes_home() + hcfg = load_config_from_hermes_home() + provider, judge_model = make_provider(hcfg, model_override=judge_model_override) + except Exception as exc: + print( + f"[warn] judge provider unavailable ({exc}); Type B falls back to regex-only", + file=sys.stderr, + ) + type_b_results = [ + await _detect_type_b( + o, + events, + provider=None if skip_b_judge else provider, + judge_model=judge_model, + ) + for o in outcomes.get("type_b_reactive_achievable") or [] + ] + # Restraint polices every agent-DECIDED delivery: sentinel nudges plus + # cron fires the agent registered on its own initiative. Executing a + # user standing order (even inside a quiet window) stays exempt — + # symmetric with Type A's decision attribution, and closes the gap + # where a self-registered 03:00 cron would earn Type A yet be + # invisible to every restraint constraint. + agent_decided_cron = [ + e + for e in delivery_events + if e.get("kind") in ("cron_fire", "hermes_cron_fire") and not _is_standing_order_fire(e, registry) + ] + restraint_events = unprompted_events + agent_decided_cron + type_c_results = [_detect_type_c(o, restraint_events, user_sends) for o in outcomes.get("type_c_restraint") or []] def _sum(rs, key="points_earned"): return sum(r.get(key, 0) or 0 for r in rs) @@ -420,34 +905,39 @@ def _sum(rs, key="points_earned"): mem_acc: dict[str, Any] = {"skipped": skip_memory_accuracy} if not skip_memory_accuracy: if provider is None: - load_env_from_hermes_home() - hcfg = load_config_from_hermes_home() - provider, judge_model = make_provider(hcfg) - # Memory.md lives inside the workspace which is already cleaned up; - # but we can reconstruct what ended up persisted from trajectory-adjacent artifacts. - # For now, rely on trajectory last memory_tail events or skip. - # MVP: we check state.json captured in ckpt; final_memory is harder post-cleanup. - # Fallback: skip with note if not found. - final_mem = _try_find_final_memory(persona_id, agent) - if final_mem: - mem_acc = await _score_memory_accuracy( - final_mem, - persona, - intents, - provider, - judge_model, - ) - else: - mem_acc = {"skipped": True, "reason": "no final MEMORY.md found; need to enable dump"} + try: + load_env_from_hermes_home() + hcfg = load_config_from_hermes_home() + provider, judge_model = make_provider(hcfg, model_override=judge_model_override) + except Exception as exc: + mem_acc = {"skipped": True, "reason": f"judge provider unavailable: {exc}"} + if provider is not None: + # Memory.md lives inside the workspace which is already cleaned + # up; reconstruct what got persisted from trajectory-adjacent + # artifacts, else skip with a note. + final_mem = _try_find_final_memory(persona_id, agent) + if final_mem: + mem_acc = await _score_memory_accuracy( + final_mem, + persona, + intents, + provider, + judge_model, + ) + else: + mem_acc = {"skipped": True, "reason": "no final MEMORY.md found; need to enable dump"} scorecard = { "persona_id": persona_id, "agent": agent, + "soft_dnd": soft_dnd, "trajectory": str(traj_path), "event_count": len(events), - "agent_initiated_count": len(agent_events), + "agent_initiated_count": len(unprompted_events), + "delivery_event_count": len(delivery_events), "user_send_count": len(user_sends), "totals": totals, + "judge_model": None if skip_b_judge or provider is None else judge_model, "proactive_lift": totals["type_a"]["earned"], # the magic number "type_a": type_a_results, "type_b": type_b_results, @@ -839,6 +1329,21 @@ def main() -> None: help="With --aggregate: path for the aggregate markdown (default: output/longrun/aggregate-scorecard.md)", ) ap.add_argument("--skip-memory", action="store_true", help="Skip LLM-backed memory accuracy scoring") + ap.add_argument( + "--skip-b-judge", + action="store_true", + help="Skip LLM confirmation of Type B exchanges (regex-only verdicts)", + ) + ap.add_argument( + "--judge-model", + default=None, + help=( + "Override the judge LLM (Type B confirmation + memory accuracy). " + "Default: the hermes-home config model. An 'openrouter/' prefix " + "re-infers OpenRouter routing, e.g. " + "openrouter/anthropic/claude-sonnet-4.5" + ), + ) args = ap.parse_args() if args.output_dir: @@ -866,7 +1371,13 @@ async def run() -> None: by_persona: dict[str, dict[str, dict]] = defaultdict(dict) for persona, ag in sorted(pairs): print(f"[score] {persona} × {ag}", file=sys.stderr) - sc = await score_trajectory(persona, ag, skip_memory_accuracy=args.skip_memory) + sc = await score_trajectory( + persona, + ag, + skip_memory_accuracy=args.skip_memory, + skip_b_judge=args.skip_b_judge, + judge_model_override=args.judge_model, + ) by_persona[persona][ag] = sc if args.compare: for persona, scs in by_persona.items(): @@ -875,7 +1386,13 @@ async def run() -> None: out.write_text(md, encoding="utf-8") print(f"[done] {out}", file=sys.stderr) elif args.persona: - await score_trajectory(args.persona, args.agent, skip_memory_accuracy=args.skip_memory) + await score_trajectory( + args.persona, + args.agent, + skip_memory_accuracy=args.skip_memory, + skip_b_judge=args.skip_b_judge, + judge_model_override=args.judge_model, + ) elif args.aggregate: pass # handled after run() else: diff --git a/benchmarks/proactivity_eval/runners/prompts/uniform/uniform_agent.yaml b/benchmarks/proactivity_eval/runners/prompts/uniform/uniform_agent.yaml new file mode 100644 index 0000000..b45507c --- /dev/null +++ b/benchmarks/proactivity_eval/runners/prompts/uniform/uniform_agent.yaml @@ -0,0 +1,25 @@ +# Uniform minimal task framing — identical for every agent (no persona, +# no decision principles). The only author-injected text in the uniform +# pbench cells; see the ablation appendix in the benchmark README. +# +# Placeholders (filled by prompts_loader.render() via str.format): +# {obs_block} — activity stream lines, always present +# {synth_block} — synthesized context; empty string in cold mode + +system: | + You are a personal AI assistant. You observe the user's recent activity + stream and decide whether to proactively reach out to the user right now. + + When you have decided, emit your final answer as a JSON code block and stop: + + ```json + {{"should_help": true|false, "proposed_task": "", "reason": ""}} + ``` + +user: | + Recent activity stream (time-ordered, most recent last): + + {obs_block} + {synth_block} + Decide whether to proactively reach out to the user right now, then emit + the JSON decision and stop. diff --git a/benchmarks/proactivity_eval/runners/run.py b/benchmarks/proactivity_eval/runners/run.py index 8b510e3..38cd953 100755 --- a/benchmarks/proactivity_eval/runners/run.py +++ b/benchmarks/proactivity_eval/runners/run.py @@ -72,6 +72,8 @@ def _backend_overrides(args: argparse.Namespace) -> dict[str, Any]: out["max_iterations"] = args.max_iter if args.agent_timeout: out["agent_timeout_s"] = args.agent_timeout + if getattr(args, "raven_config", None): + out["raven_config"] = args.raven_config # Shared memory toggle (honored by Hermes + OpenClaw backends) if args.with_memory: out["with_memory"] = True @@ -102,6 +104,7 @@ def _build_driver(args: argparse.Namespace) -> BenchmarkDriver: agent_name=args.agent, context_mode=args.context_mode, synthesizer_name=args.synthesizer, + prompts_dir=Path(args.prompts_dir) if getattr(args, "prompts_dir", None) else None, ) if args.benchmark == "cases": return get_driver("cases") @@ -314,6 +317,12 @@ def main() -> None: # pbench-specific ap.add_argument("--context-mode", choices=["cold", "warm"], default="cold") ap.add_argument("--synthesizer", default="keyword") + ap.add_argument( + "--prompts-dir", + default=None, + help="pbench: directory of _agent.yaml prompt templates " + "(default: runners/prompts/). For prompt-swap ablations.", + ) # openclaw-specific ap.add_argument("--thinking", default=None, choices=["off", "minimal", "low", "medium", "high", "xhigh"]) @@ -334,6 +343,13 @@ def main() -> None: # raven-specific ap.add_argument("--max-iter", type=int, default=None, help="Raven AgentLoop max_iterations") ap.add_argument("--agent-timeout", type=int, default=None, help="Raven agent per-sample wall timeout (s)") + ap.add_argument( + "--raven-config", + default=None, + help="Raven: --config passed to every raven subprocess (also " + "redirects the raven data dir to the config's parent). For " + "model/endpoint ablations without touching ~/.raven.", + ) # shared ap.add_argument(