From c266643ab5824ccda81c7d81ed5de8b431b1cfdc Mon Sep 17 00:00:00 2001 From: khanhphamj Date: Sat, 31 Oct 2026 10:16:38 +0700 Subject: [PATCH 1/2] Held-out set 3 and a recorded negative result for Haiku bare-question routing Two description rewordings (assume-AgentBase sentence, general-question clause) were measured on a fresh 36-case set and did not move Haiku 5.5 (75% -> 76%), so descriptions are unchanged; the finding and the new set are documented. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 ++ evals/README.md | 8 ++++++++ evals/triggers_heldout3.jsonl | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 evals/triggers_heldout3.jsonl diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b999b7..ca47060 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ review can tell which standard a project was built from. - **Routing eval for non-Claude models:** `evals/run_triggers_api.py` (GPT, DeepSeek, GLM, Gemma, Kimi, MiniMax, Qwen through any OpenAI-compatible API, simulated skill listing) and a second held-out set; results per model in `evals/README.md`. Known weak spot: small models skip skills for a bare question that does not name AgentBase (Haiku 5.5 64% on held-out2). +- `evals/triggers_heldout3.jsonl` (36 cases) and a recorded negative result: two description rewordings did not raise Haiku 5.5 + on bare questions (75% → 76%), so descriptions are unchanged; see `evals/README.md`. ## 0.5.0 — 2026-10-29 diff --git a/evals/README.md b/evals/README.md index a2a8f6e..479afdd 100644 --- a/evals/README.md +++ b/evals/README.md @@ -104,6 +104,14 @@ What this showed: AgentBase.", score 86%. Inside a generated project `AGENTS.md` carries that context (in-project cases pass), so the weak spot is a bare question in an empty directory on a small model. Name AgentBase in the request, or work in a generated project. +- **Wording does not fix it (tried, reverted).** A third set, `triggers_heldout3.jsonl` (36 cases, half inside a generated + project, half bare), scored 75% on Haiku 5.5 as written (2 runs per case). Two description changes were measured on it and + gave nothing, so they were not kept: adding "Assume it is a GreenNode AgentBase agent even if the user does not say so" to + 15 descriptions (76%), and adding "a general or how-to question included" to the load clause (76%). Every miss is the same + shape: a bare English question about "my agent" in an empty directory (memory, tracing, human approval, MCP, web chat page) + that Haiku answers itself; the cases inside a project, and the Vietnamese ones that name a concrete tool or endpoint, pass. + The remaining lever is the request or the project, not the description. Because that set drove those experiments it is + now training data; a fresh set is needed before another description change is judged. - **Where it still misses:** `h2-eval-q` and `h2-llm-q` ("how do I know my last prompt change didn't make answers worse?", "which model for the router?") → no skill, on Sonnet too; `agentbase-build-mcp` vs `agentbase-build-mcp-server` on GPT-5.4 nano; `claude-api` takes prompt-quality requests on Claude. diff --git a/evals/triggers_heldout3.jsonl b/evals/triggers_heldout3.jsonl new file mode 100644 index 0000000..754b1a3 --- /dev/null +++ b/evals/triggers_heldout3.jsonl @@ -0,0 +1,36 @@ +{"id": "h3-tracing-q", "prompt": "Which step of my agent's pipeline is the slowest? I'd like a timeline for one request.", "expect": ["agentbase-build-tracing"]} +{"id": "h3-tracing-vi", "prompt": "Nhìn trace của một lượt chat để biết tool nào gọi lâu nhất.", "expect": ["agentbase-build-tracing"], "in_project": true} +{"id": "h3-hitl-q", "prompt": "Before the agent sends an email to a customer, a human on our team should approve it.", "expect": ["agentbase-build-hitl"]} +{"id": "h3-hitl-vi", "prompt": "Cho agent dừng lại hỏi mình trước khi chạy tool refund.", "expect": ["agentbase-build-hitl"], "in_project": true} +{"id": "h3-ident-q", "prompt": "My agent needs to call the Jira API on behalf of each employee, how do I store their tokens?", "expect": ["agentbase-build-identity"]} +{"id": "h3-ident-vi", "prompt": "Chỗ nào để lưu API key của SendGrid cho tool gửi mail?", "expect": ["agentbase-build-identity"], "in_project": true} +{"id": "h3-sec-q", "prompt": "Review this agent for secrets in the repo, SSRF and unsafe tool inputs.", "expect": ["agentbase-build-security-audit"], "in_project": true} +{"id": "h3-sec-vi", "prompt": "Lập checklist pen-test cho agent trước khi team bảo mật duyệt.", "expect": ["agentbase-build-security-audit"]} +{"id": "h3-mcpserver-q", "prompt": "Package our CRM lookup as an MCP server that other agents can connect to.", "expect": ["agentbase-build-mcp-server"]} +{"id": "h3-mcpserver-vi", "prompt": "Viết MCP server expose 3 hàm tra cứu đơn hàng, có auth bằng API key.", "expect": ["agentbase-build-mcp-server"]} +{"id": "h3-mcp-q", "prompt": "Connect the agent to our existing GitHub MCP server so it can open issues.", "expect": ["agentbase-build-mcp"]} +{"id": "h3-mcp-vi", "prompt": "Agent không thấy tool nào từ MCP server, kiểm tra giúp.", "expect": ["agentbase-build-mcp"], "in_project": true} +{"id": "h3-memory-q", "prompt": "Remember the user's preferred language across sessions.", "expect": ["agentbase-build-memory"], "in_project": true} +{"id": "h3-memory-vi", "prompt": "Agent nhớ tên khách hàng từ tuần trước được không?", "expect": ["agentbase-build-memory"]} +{"id": "h3-auth-q", "prompt": "Only logged-in employees from our Keycloak should be able to call the agent.", "expect": ["agentbase-build-auth"]} +{"id": "h3-auth-vi", "prompt": "Thêm đăng nhập bằng JWT cho endpoint /invocations.", "expect": ["agentbase-build-auth"], "in_project": true} +{"id": "h3-eval-q", "prompt": "Set up a regression test with 30 sample questions I can run before each release.", "expect": ["agentbase-build-eval"], "in_project": true} +{"id": "h3-eval-vi", "prompt": "Chấm điểm câu trả lời của agent bằng LLM judge, chạy trong CI.", "expect": ["agentbase-build-eval"]} +{"id": "h3-llm-q", "prompt": "Swap the model the agent uses to a cheaper one for simple questions.", "expect": ["agentbase-build-llm"], "in_project": true} +{"id": "h3-llm-vi", "prompt": "Đổi model của agent sang Qwen, cấu hình ở đâu?", "expect": ["agentbase-build-llm"], "in_project": true} +{"id": "h3-deploy-q", "prompt": "Push the agent image and create the runtime.", "expect": ["agentbase-build-deploy", "agentbase-deploy"], "in_project": true} +{"id": "h3-deploy-vi", "prompt": "Deploy lên GreenNode giúp mình, sau đó kiểm tra /health.", "expect": ["agentbase-build-deploy", "agentbase-deploy"], "in_project": true} +{"id": "h3-web-q", "prompt": "Add a browser chat page for the agent that supports streaming and login.", "expect": ["agentbase-build-web"]} +{"id": "h3-web-vi", "prompt": "Làm trang web cho nhân viên chat với agent.", "expect": ["agentbase-build-web"]} +{"id": "h3-frontend-q", "prompt": "Build an Android app that talks to the agent.", "expect": ["agentbase-build-frontend"]} +{"id": "h3-frontend-vi", "prompt": "Cần app mobile Expo cho agent, có đăng nhập.", "expect": ["agentbase-build-frontend"]} +{"id": "h3-a2a-q", "prompt": "Let our agent be called by another team's agent over A2A.", "expect": ["agentbase-build-a2a"], "in_project": true} +{"id": "h3-a2a-vi", "prompt": "Cho agent gọi một agent khác qua giao thức A2A.", "expect": ["agentbase-build-a2a"]} +{"id": "h3-scaffold-q", "prompt": "Start a new agent project with the standard folders and a Dockerfile.", "expect": ["agentbase-build-scaffold", "agentbase-build"]} +{"id": "h3-scaffold-vi", "prompt": "Khởi tạo dự án agent mới tên support-bot.", "expect": ["agentbase-build-scaffold", "agentbase-build"]} +{"id": "h3-build-q", "prompt": "Build me an AI agent that answers HR policy questions for employees.", "expect": ["agentbase-build"]} +{"id": "h3-build-vi", "prompt": "Tôi muốn làm một agent chăm sóc khách hàng trên nền tảng GreenNode.", "expect": ["agentbase-build"]} +{"id": "h3-neg-logs", "prompt": "Tail the logs of my running agent, it just crashed.", "expect": ["agentbase-monitor"]} +{"id": "h3-neg-models", "prompt": "Which LLM models can my API key access on the platform?", "expect": ["agentbase-llm"]} +{"id": "h3-neg-teardown", "prompt": "Remove all the cloud resources of this demo agent.", "expect": ["agentbase-teardown"]} +{"id": "h3-neg-none", "prompt": "Explain the difference between a thread and a process.", "expect": ["none"]} From 0d8757dbe5ded861c68d5187bed8610b0feff647 Mon Sep 17 00:00:00 2001 From: khanhphamj Date: Sat, 31 Oct 2026 10:52:56 +0700 Subject: [PATCH 2/2] SessionStart hook: 'my agent' means an AgentBase agent (Claude Code); held-out set 4 Haiku 5.5 on bare questions 66% -> 92% (fresh set), 64% -> 95% (held-out2), tuning 95% -> 100%. The last wording tweak (skip agents on another named stack) is not re-measured: weekly limit reached. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 ++++-- evals/README.md | 18 ++++++++++++++-- evals/triggers_heldout4.jsonl | 40 +++++++++++++++++++++++++++++++++++ hooks/hooks.json | 12 +++++++++++ hooks/session-context.txt | 1 + 5 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 evals/triggers_heldout4.jsonl create mode 100644 hooks/hooks.json create mode 100644 hooks/session-context.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index ca47060..ed5a786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,10 @@ review can tell which standard a project was built from. - **Routing eval for non-Claude models:** `evals/run_triggers_api.py` (GPT, DeepSeek, GLM, Gemma, Kimi, MiniMax, Qwen through any OpenAI-compatible API, simulated skill listing) and a second held-out set; results per model in `evals/README.md`. Known weak spot: small models skip skills for a bare question that does not name AgentBase (Haiku 5.5 64% on held-out2). -- `evals/triggers_heldout3.jsonl` (36 cases) and a recorded negative result: two description rewordings did not raise Haiku 5.5 - on bare questions (75% → 76%), so descriptions are unchanged; see `evals/README.md`. +- **Claude Code only: SessionStart hook** (`hooks/`) tells the model that "my agent" means a GreenNode AgentBase agent unless + another framework or platform is named. Haiku 5.5 on bare questions: 66% → 92% (fresh held-out set), 64% → 95% (held-out2), + tuning set 95% → 100%. Descriptions are unchanged (two rewordings gave nothing). The last wording tweak (do not claim + CrewAI / AWS agents) is not re-measured yet; see `evals/README.md`. New sets: `triggers_heldout3.jsonl`, `triggers_heldout4.jsonl`. ## 0.5.0 — 2026-10-29 diff --git a/evals/README.md b/evals/README.md index 479afdd..0fe082e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -110,8 +110,22 @@ What this showed: 15 descriptions (76%), and adding "a general or how-to question included" to the load clause (76%). Every miss is the same shape: a bare English question about "my agent" in an empty directory (memory, tracing, human approval, MCP, web chat page) that Haiku answers itself; the cases inside a project, and the Vietnamese ones that name a concrete tool or endpoint, pass. - The remaining lever is the request or the project, not the description. Because that set drove those experiments it is - now training data; a fresh set is needed before another description change is judged. + What does work in Claude Code is context the plugin puts in the session itself: `hooks/hooks.json` (SessionStart) injects + `hooks/session-context.txt`, three lines saying that "my agent" means a GreenNode AgentBase agent unless another + framework or platform is named, and to load the matching `agentbase-build-*` skill. Haiku 5.5, 2 runs per case: + + | Set | without the hook | with the hook | + |---|---|---| + | `triggers_heldout3.jsonl` (36; used while trying the hook) | 75% | 92% | + | `triggers_heldout4.jsonl` (40, written before the hook was measured; 8 of them name another stack, e.g. CrewAI, Lambda) | 66% | 92% | + | `triggers.jsonl` (58, tuning) | 95% | 100% | + | `triggers_heldout2.jsonl` (32) | 64% | 95% | + + All 8 other-stack cases stayed `none` on Haiku. Sonnet 5.5 on held-out4 with the hook: 98% — its one miss was "Add long-term + memory to my CrewAI agent" → `agentbase-build-memory`, which is why the text now says "unless they name another framework + or platform". **That wording change was not re-measured**: Claude Code's weekly limit was reached right after, so the final + text has no score yet (rerun held-out4 on Sonnet and Haiku, and the tuning set on Sonnet). Only Claude Code runs the hook; + Codex and the other agents do not, so the numbers for those stay as in the table above. - **Where it still misses:** `h2-eval-q` and `h2-llm-q` ("how do I know my last prompt change didn't make answers worse?", "which model for the router?") → no skill, on Sonnet too; `agentbase-build-mcp` vs `agentbase-build-mcp-server` on GPT-5.4 nano; `claude-api` takes prompt-quality requests on Claude. diff --git a/evals/triggers_heldout4.jsonl b/evals/triggers_heldout4.jsonl new file mode 100644 index 0000000..ceacdf8 --- /dev/null +++ b/evals/triggers_heldout4.jsonl @@ -0,0 +1,40 @@ +{"id": "h4-tracing-q", "prompt": "Our agent's answers got slower this week. Where is the time going per request?", "expect": ["agentbase-build-tracing"]} +{"id": "h4-tracing-vi", "prompt": "Mỗi câu trả lời của agent tốn bao nhiêu token, xem ở đâu?", "expect": ["agentbase-build-tracing"]} +{"id": "h4-hitl-q", "prompt": "I want a manager to sign off before the agent issues a refund.", "expect": ["agentbase-build-hitl"]} +{"id": "h4-hitl-vi", "prompt": "Agent tự gửi mail cho khách, làm sao để nó hỏi mình duyệt trước?", "expect": ["agentbase-build-hitl"]} +{"id": "h4-ident-q", "prompt": "Where should the agent keep the Salesforce OAuth token for each user?", "expect": ["agentbase-build-identity"]} +{"id": "h4-ident-vi", "prompt": "Tool của agent cần key của Stripe, cất key ở đâu cho an toàn?", "expect": ["agentbase-build-identity"]} +{"id": "h4-sec-q", "prompt": "Can someone trick my agent into leaking the system prompt? Audit it.", "expect": ["agentbase-build-security-audit"]} +{"id": "h4-sec-vi", "prompt": "Kiểm tra bảo mật agent trước khi mở cho khách hàng dùng.", "expect": ["agentbase-build-security-audit"]} +{"id": "h4-mcpserver-q", "prompt": "Turn our shipping-rates API into an MCP server my agents can call.", "expect": ["agentbase-build-mcp-server"]} +{"id": "h4-mcpserver-vi", "prompt": "Tôi cần viết server MCP bọc API kho hàng, bảo vệ bằng OAuth.", "expect": ["agentbase-build-mcp-server"]} +{"id": "h4-mcp-q", "prompt": "Let the agent read and create Notion pages through MCP.", "expect": ["agentbase-build-mcp"]} +{"id": "h4-mcp-vi", "prompt": "Agent gọi tool báo denied by policy, sửa thế nào?", "expect": ["agentbase-build-mcp"]} +{"id": "h4-memory-q", "prompt": "The agent should recall what a customer asked last month.", "expect": ["agentbase-build-memory"]} +{"id": "h4-memory-vi", "prompt": "Chat dài thì agent quên đoạn đầu, khắc phục sao?", "expect": ["agentbase-build-memory"]} +{"id": "h4-auth-q", "prompt": "Require a login before anyone can call the agent's endpoint.", "expect": ["agentbase-build-auth"]} +{"id": "h4-auth-vi", "prompt": "Chặn người lạ gọi agent, chỉ cho nhân viên đăng nhập bằng Azure AD.", "expect": ["agentbase-build-auth"]} +{"id": "h4-eval-q", "prompt": "I need a way to score the agent's answers automatically after every change.", "expect": ["agentbase-build-eval"]} +{"id": "h4-eval-vi", "prompt": "Làm bộ test chất lượng câu trả lời cho agent, chạy trước khi release.", "expect": ["agentbase-build-eval"]} +{"id": "h4-llm-q", "prompt": "Use a bigger model for hard questions and a small one for greetings in my agent.", "expect": ["agentbase-build-llm"]} +{"id": "h4-llm-vi", "prompt": "Agent báo LLM 429 liên tục, chuyển sang model dự phòng thế nào?", "expect": ["agentbase-build-llm"]} +{"id": "h4-deploy-q", "prompt": "Release the agent to production and roll back if the health check fails.", "expect": ["agentbase-build-deploy", "agentbase-deploy"]} +{"id": "h4-deploy-vi", "prompt": "Đưa agent lên môi trường staging giúp mình.", "expect": ["agentbase-build-deploy", "agentbase-deploy"]} +{"id": "h4-web-q", "prompt": "Give the agent a chat website for our customers.", "expect": ["agentbase-build-web"]} +{"id": "h4-frontend-q", "prompt": "Make a phone app so field staff can talk to the agent.", "expect": ["agentbase-build-frontend"]} +{"id": "h4-a2a-q", "prompt": "My agent should delegate shipping questions to the logistics team's agent.", "expect": ["agentbase-build-a2a"]} +{"id": "h4-scaffold-q", "prompt": "Set up the project structure for a new agent called invoice-bot.", "expect": ["agentbase-build-scaffold", "agentbase-build"]} +{"id": "h4-build-q", "prompt": "I want an assistant that answers questions about our product catalog.", "expect": ["agentbase-build"]} +{"id": "h4-build-vi", "prompt": "Làm giúp mình agent tư vấn gói bảo hiểm cho khách.", "expect": ["agentbase-build"]} +{"id": "h4-neg-logs", "prompt": "Show the recent logs of my deployed agent, something is failing.", "expect": ["agentbase-monitor"]} +{"id": "h4-neg-models", "prompt": "List the models I can call with my platform API key.", "expect": ["agentbase-llm"]} +{"id": "h4-neg-teardown", "prompt": "Wipe every resource I created for the test agent.", "expect": ["agentbase-teardown"]} +{"id": "h4-neg-none", "prompt": "How do I reverse a linked list in Java?", "expect": ["none"]} +{"id": "h4-other-crewai", "prompt": "Add long-term memory to my CrewAI agent.", "expect": ["none"]} +{"id": "h4-other-aws", "prompt": "Deploy my LangChain agent to AWS Lambda.", "expect": ["none"]} +{"id": "h4-other-openai", "prompt": "My OpenAI Assistants API agent loses context between runs, why?", "expect": ["none"]} +{"id": "h4-other-concept", "prompt": "Explain how a ReAct agent decides which tool to call.", "expect": ["none"]} +{"id": "h4-other-evals", "prompt": "Which metrics should I use to evaluate a RAG pipeline?", "expect": ["none"]} +{"id": "h4-other-auth", "prompt": "Add JWT login to my Express.js API.", "expect": ["none"]} +{"id": "h4-other-mcp", "prompt": "Write an MCP server in TypeScript for my Claude Desktop that reads local markdown files.", "expect": ["none"]} +{"id": "h4-other-web", "prompt": "Add a dark mode toggle to my React website.", "expect": ["none"]} diff --git a/hooks/hooks.json b/hooks/hooks.json new file mode 100644 index 0000000..bb30a25 --- /dev/null +++ b/hooks/hooks.json @@ -0,0 +1,12 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|clear", + "hooks": [ + { "type": "command", "command": "cat \"${CLAUDE_PLUGIN_ROOT}/hooks/session-context.txt\"" } + ] + } + ] + } +} diff --git a/hooks/session-context.txt b/hooks/session-context.txt new file mode 100644 index 0000000..8c25fdf --- /dev/null +++ b/hooks/session-context.txt @@ -0,0 +1 @@ +The agentbase-build skills are installed. When the user asks about an AI agent of theirs ("my agent", "the agent") — its memory, tracing or token cost, human approval of actions, tools or MCP, credentials for external services, login or auth, evals, which LLM, deploy, a web or mobile chat UI, calling other agents, security — assume it is a GreenNode AgentBase agent even if they do not say so (unless they name another framework or platform, such as CrewAI, AWS or the OpenAI Assistants API), and load the matching agentbase-build-* skill before answering.