Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions docs/adr-008-harness-event-drift.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,31 @@ Reader 靠比對字面字串認出 Claude Code 寫進 transcript 的事件。那

量測基礎:`~/.claude/projects` 下依修改時間取最新 120 個 `.jsonl`(2026-08-30 往前 60 天)。

**追加樣本(2026-09-02)**:原始盤點跳過了 `<session>/subagents/*.jsonl` 這一層,
這次先把最大的 120 個 subagent transcript 併入盤點,找到五個沿用既有決定即可處理的樣本:

- `relocated` entry type(600 筆/82 KB),併入第 1 項的白名單。
- `<task-notification>` 前面多了一段「[SYSTEM NOTIFICATION - NOT USER INPUT]」免責聲明
(68 則,CLI 2.1.200–2.1.235,全在 subagent transcript),偵測改成認標籤本身而非開頭前綴,
沿用第 4 項認標籤不認散文的理由。
- `The coordinator sent a message while you were working:`(3 則),coordinator 對 subagent
發起一輪工作,比照第 2、5 項歸類為 harness、算 turn。
- `<local-command-stderr>`(2 則),跟已處理的 stdout 變體同一種病,補上同樣的處理。
- `Continue from where you left off.`(2 則,帶頂層 `isMeta: true`),
是別處已啟動的 invocation 的尾巴,比照 Stop hook 通知不算 turn。

再把盤點範圍擴大到主 session 與 subagent 兩層合併掃描,又找到三個:

- `<fork-boilerplate>`…`</fork-boilerplate>`(5 則,全在 subagent transcript,約 1,000 字元),
worker fork 的固定開場白,壓成 `[fork]`,閉合標籤後的內文(fork 的實際指令)保留;算 turn,
因為它啟動了這個 fork 的工作。
- `[Your previous response had no visible output. Please continue and produce a user-visible response.]`
(36 則,精確文字,主 session),壓成 `[nudge: no visible output]`;算 turn,
因為它是要求新一輪回應,不是在報告已經發生的事。
- `The user sent a new message while you were working:` 開頭、接一段解釋文字收尾(35 則),
內文本身是人打的字,所以維持 `user:` 標籤,只剝掉開頭與結尾的 harness 說明文字;
不算 turn,因為 harness 自己的說明就寫著這則訊息落在正在跑的那一輪裡面,不是另開一輪。

## 1. `noiseTypes` 漏了 8 個 CLI 後來才加的 entry type

`noiseTypes` 是手寫的白名單,收 13 個型別。實測出現、不在名單裡的有 8 個:
Expand Down
4 changes: 3 additions & 1 deletion internal/analyzer/stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ func ComputeStats(events []session.Event) StatsResult {
if event.User.IsSkillInjection || event.User.IsTeammateMessage ||
event.User.IsCommandInjection || event.User.IsTaskNotification ||
event.User.IsCompactionSummary || event.User.IsStopHookGoal ||
event.User.IsAgentsStopped || event.User.IsInterrupted {
event.User.IsAgentsStopped || event.User.IsInterrupted ||
event.User.IsCoordinatorMessage || event.User.IsContinuePrompt ||
event.User.IsForkBoilerplate || event.User.IsNoVisibleOutputNudge {
rawParts = append(rawParts, event.User.Text)
continue
}
Expand Down
97 changes: 89 additions & 8 deletions internal/claudecodec/classify.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ const (
tagBashInputOpen = "<bash-input>"
tagBashInputClose = "</bash-input>"
tagLocalStdout = "<local-command-stdout>"
tagLocalStderr = "<local-command-stderr>"
tagBashStdout = "<bash-stdout>"
tagBashStderr = "<bash-stderr>"
tagLocalCaveat = "<local-command-caveat>"
Expand All @@ -37,11 +38,33 @@ const (
commandMessageOpen = "<command-message>"
skillArgsPrefix = "ARGUMENTS:"

taskNotificationOpen = "<task-notification>"
compactionSummary = "This session is being continued from a previous conversation"
interruptedPrefix = "[Request interrupted by user"
stopHookPrefix = "A session-scoped Stop hook is now active with condition:"
skillReloadedMarker = "was loaded earlier"
taskNotificationOpen = "<task-notification>"
compactionSummary = "This session is being continued from a previous conversation"
interruptedPrefix = "[Request interrupted by user"
stopHookPrefix = "A session-scoped Stop hook is now active with condition:"
skillReloadedMarker = "was loaded earlier"
coordinatorMessageOpen = "The coordinator sent a message while you were working:"

// continuePromptText is the exact harness-injected body that resumes an
// invocation already started elsewhere. It carries no sourceToolUseID
// link the way a skill injection does — only isMeta at the top level —
// so it is matched on exact text rather than a prefix, the same way the
// stop-hook goal is matched by its fixed wording.
continuePromptText = "Continue from where you left off."

forkBoilerplateOpen = "<fork-boilerplate>"
forkBoilerplateClose = "</fork-boilerplate>"

// noVisibleOutputNudge is the exact harness nudge sent when an assistant
// turn produced no visible output.
noVisibleOutputNudge = "[Your previous response had no visible output. Please continue and produce a user-visible response.]"

// midTurnOpeningLine and midTurnExplanationMarker bracket the human text
// in a mid-turn message notice: the harness wraps a message the user sent
// while the agent was still working in an explanation of when it arrives,
// but the body between them is exactly what the user typed.
midTurnOpeningLine = "The user sent a new message while you were working:"
midTurnExplanationMarker = "This is how Claude Code surfaces messages the user sends mid-turn"
)

var agentsStoppedCount = regexp.MustCompile(`^(\d+) background agents? (?:was|were) stopped`)
Expand Down Expand Up @@ -82,9 +105,11 @@ func classifyCommandUserMessage(text string) *session.UserMessage {
}
}

// Command output (slash stdout, bash stdout/stderr): droppable body,
// surfaced only under -verbose-commands with ANSI stripped at render time.
// Command output (slash stdout/stderr, bash stdout/stderr): droppable
// body, surfaced only under -verbose-commands with ANSI stripped at
// render time.
if strings.HasPrefix(trimmed, tagLocalStdout) ||
strings.HasPrefix(trimmed, tagLocalStderr) ||
strings.HasPrefix(trimmed, tagBashStdout) ||
strings.HasPrefix(trimmed, tagBashStderr) {
return &session.UserMessage{IsCommandNoise: true, Text: trimmed}
Expand Down Expand Up @@ -121,6 +146,17 @@ func classifySkillInjectionByLink(text string, isMeta bool, sourceToolUseID stri
}
}

// classifyContinuePrompt detects the exact-text isMeta continuation prompt
// that resumes an invocation already started elsewhere. isMeta alone is not
// a marker (image placeholders and stop-hook feedback also carry it, see
// classifySkillInjectionByLink), so this also requires the exact text.
func classifyContinuePrompt(text string, isMeta bool) *session.UserMessage {
if !isMeta || strings.TrimSpace(text) != continuePromptText {
return nil
}
return &session.UserMessage{Text: text, IsContinuePrompt: true}
}

// classifyHarnessUserMessage detects harness-injected user messages that are
// not direct user input: skill injections, system reminders, teammate messages,
// context usage blocks, and command injection XML. Returns nil for plain
Expand All @@ -145,6 +181,16 @@ func classifyHarnessUserMessage(text string) *session.UserMessage {
}
}

// Mid-turn user message: the body is genuinely what the user typed, sent
// while the agent was still working on the previous turn, so it falls
// back to plain user-message handling for everything except the
// wrapper text stripped here.
if strings.HasPrefix(trimmed, midTurnOpeningLine) {
if body, ok := extractMidTurnUserText(trimmed); ok {
return &session.UserMessage{Text: text, IsMidTurnUserMessage: true, MidTurnUserText: body}
}
}

// Teammate message: detected by the XML tag alone, not by the surrounding
// prose ("Another Claude session sent a message:", the disclaimer). Both
// have already been reworded once without the tag changing, and the
Expand All @@ -171,10 +217,32 @@ func classifyHarnessUserMessage(text string) *session.UserMessage {

// Background-task report. Recognized here rather than only at render time
// so stats sees a domain field like every other subtype (ADR-008).
if strings.HasPrefix(trimmed, taskNotificationOpen) {
// Matched by Contains, not HasPrefix: a CLI build wraps the tag in a
// "[SYSTEM NOTIFICATION - NOT USER INPUT]" disclaimer that precedes it,
// the same drift the teammate tag detection already accounts for.
if strings.Contains(trimmed, taskNotificationOpen) {
return &session.UserMessage{Text: text, IsTaskNotification: true}
}

// Coordinator-initiated round of work in a subagent transcript: the
// coordinator's own message starts a turn the same way a teammate
// message does, so it gets the same treatment.
if strings.HasPrefix(trimmed, coordinatorMessageOpen) {
return &session.UserMessage{Text: text, IsCoordinatorMessage: true}
}

// Worker-fork preamble: the harness's instructions for a forked worker,
// followed by the actual directive text (if any) after the closing tag.
if strings.HasPrefix(trimmed, forkBoilerplateOpen) {
return &session.UserMessage{Text: text, IsForkBoilerplate: true}
}

// Fixed nudge sent when the previous assistant turn had no visible
// output. Exact text, so no extraction needed.
if trimmed == noVisibleOutputNudge {
return &session.UserMessage{Text: text, IsNoVisibleOutputNudge: true}
}

// Conversation summary injected when a session continues past a
// compaction. The body is the previous conversation, so it is kept.
if strings.HasPrefix(trimmed, compactionSummary) {
Expand Down Expand Up @@ -230,6 +298,19 @@ func extractGoalCondition(text string) string {
return rest[:end]
}

// extractMidTurnUserText pulls the human-typed body out of a mid-turn user
// message notice, stripping the opening line and the trailing explanation
// paragraph. Returns ("", false) if the explanation marker is absent, so the
// caller does not classify a message whose shape it can't fully account for.
func extractMidTurnUserText(text string) (string, bool) {
rest := strings.TrimPrefix(text, midTurnOpeningLine)
explIdx := strings.Index(rest, midTurnExplanationMarker)
if explIdx < 0 {
return "", false
}
return strings.TrimSpace(rest[:explIdx]), true
}

// reloadedSkillName extracts the skill from a "Skill /foo was loaded earlier"
// notice, or reports false when text is not one.
func reloadedSkillName(text string) (string, bool) {
Expand Down
120 changes: 120 additions & 0 deletions internal/claudecodec/harness_classify_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,123 @@ func TestClassifyHarnessUserMessage_GivenPlainMessage_WhenClassified_ThenReturns
}
}

// The disclaimer wraps the tag in a CLI build seen only in subagent
// transcripts (68 messages, CLI 2.1.200-2.1.235); HasPrefix missed all of
// them since the tag no longer opens the message.
func TestClassifyHarnessUserMessage_GivenTaskNotificationWrappedInDisclaimer_WhenClassified_ThenStillDetectsIt(t *testing.T) {
text := "[SYSTEM NOTIFICATION - NOT USER INPUT]\n" +
"This is an automated background-task event, NOT a message from the user.\n" +
"Do NOT interpret this as user acknowledgement, confirmation, or response to any pending question.\n\n" +
"<task-notification>\n<task-id>a3d084a486cbf8046</task-id>\n" +
"<summary>Build finished</summary>\n</task-notification>"

got := classifyHarnessUserMessage(text)

if got == nil || !got.IsTaskNotification {
t.Fatalf("classifyHarnessUserMessage() = %+v, want IsTaskNotification = true", got)
}
}

// 3 messages in subagent transcripts: the coordinator opens a round of work
// the same way a teammate message does, but with different framing.
func TestClassifyHarnessUserMessage_GivenCoordinatorMessage_WhenClassified_ThenMarksIt(t *testing.T) {
text := "The coordinator sent a message while you were working:\nplease also update the README"

got := classifyHarnessUserMessage(text)

if got == nil || !got.IsCoordinatorMessage {
t.Fatalf("classifyHarnessUserMessage() = %+v, want IsCoordinatorMessage = true", got)
}
}

// classifyContinuePrompt requires the isMeta flag in addition to the exact
// text, since isMeta alone also covers image placeholders and stop-hook
// feedback (see classifySkillInjectionByLink).
func TestClassifyContinuePrompt_GivenExactTextAndIsMeta_WhenClassified_ThenMarksIt(t *testing.T) {
tests := map[string]struct {
text string
isMeta bool
want bool
}{
"exact text with isMeta true is a continue prompt": {
text: "Continue from where you left off.", isMeta: true, want: true,
},
"exact text without isMeta is not, since isMeta alone is not distinctive": {
text: "Continue from where you left off.", isMeta: false, want: false,
},
"isMeta true with different text is not": {
text: "Continue from where you left off, please.", isMeta: true, want: false,
},
}

for name, tc := range tests {
t.Run(name, func(t *testing.T) {
got := classifyContinuePrompt(tc.text, tc.isMeta)
if (got != nil) != tc.want {
t.Errorf("classifyContinuePrompt(%q, %v) = %+v, want non-nil = %v", tc.text, tc.isMeta, got, tc.want)
}
if tc.want && !got.IsContinuePrompt {
t.Errorf("IsContinuePrompt = false, want true")
}
})
}
}

// 5 messages, all subagent transcripts: the worker-fork preamble is fixed
// boilerplate, and any text after the closing tag is the fork's directive.
func TestClassifyHarnessUserMessage_GivenForkBoilerplate_WhenClassified_ThenMarksIt(t *testing.T) {
text := "<fork-boilerplate>\nYou are a worker fork. The transcript above is the parent's " +
"history. Execute ONE directive, then stop.\n</fork-boilerplate>\nFix the failing test."

got := classifyHarnessUserMessage(text)

if got == nil || !got.IsForkBoilerplate {
t.Fatalf("classifyHarnessUserMessage() = %+v, want IsForkBoilerplate = true", got)
}
}

// 36 messages, exact text, main sessions: the harness's nudge to produce a
// visible response after a silent turn.
func TestClassifyHarnessUserMessage_GivenNoVisibleOutputNudge_WhenClassified_ThenMarksIt(t *testing.T) {
text := "[Your previous response had no visible output. Please continue and produce a user-visible response.]"

got := classifyHarnessUserMessage(text)

if got == nil || !got.IsNoVisibleOutputNudge {
t.Fatalf("classifyHarnessUserMessage() = %+v, want IsNoVisibleOutputNudge = true", got)
}
}

// 35 messages: the body is human-typed, so classification must keep it under
// the user role (not harness) while still stripping the harness's wrapper.
func TestClassifyHarnessUserMessage_GivenMidTurnUserMessage_WhenClassified_ThenExtractsTheUserText(t *testing.T) {
text := "The user sent a new message while you were working:\n直接改 bug 就好\n\n" +
"This is how Claude Code surfaces messages the user sends mid-turn — within the running " +
"turn, often alongside the next tool result, rather than as a separate conversation turn. " +
"Address the message above as you continue this turn."

got := classifyHarnessUserMessage(text)

if got == nil || !got.IsMidTurnUserMessage {
t.Fatalf("classifyHarnessUserMessage() = %+v, want IsMidTurnUserMessage = true", got)
}
if want := "直接改 bug 就好"; got.MidTurnUserText != want {
t.Errorf("MidTurnUserText = %q, want %q", got.MidTurnUserText, want)
}
}

// 2 messages render as "user:" because the stderr variant of the
// local-command tag has no handling, unlike its stdout sibling.
func TestClassifyCommandUserMessage_GivenLocalCommandStderr_WhenClassified_ThenMarksItCommandNoise(t *testing.T) {
text := "<local-command-stderr>permission denied</local-command-stderr>"

got := classifyCommandUserMessage(text)

if got == nil || !got.IsCommandNoise {
t.Fatalf("classifyCommandUserMessage(%q) = %+v, want IsCommandNoise = true", text, got)
}
}

// These entry types were added to Claude Code after the original noise list.
// Without an entry they fell through as unparsed rather than as EventNoise,
// which left their bytes out of the analyzer's system_noise accounting.
Expand All @@ -160,6 +277,9 @@ func TestParseLine_GivenRecentlyAddedEntryType_WhenParsed_ThenYieldsNoise(t *tes
"atis-latch", "frame-link", "worktree-state", "file-history-delta",
"artifact-autoreact-ledger", "artifact-comment-monitor",
"agent-setting", "cost-state",
// Found when the ADR-008 scan was extended to the subagent
// transcript layer: 600 entries / 82 KB in the same 60-day window.
"relocated",
}

for _, entryType := range types {
Expand Down
8 changes: 7 additions & 1 deletion internal/claudecodec/reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ var noiseTypes = map[string]bool{
"artifact-comment-monitor": true,
"agent-setting": true,
"cost-state": true,

// Added after the ADR-008 scan was extended to the subagent transcript
// layer; observed 600 entries / 82 KB in the same 60-day window.
"relocated": true,
}

func ReadFile(path string, handle func(session.Event) error) error {
Expand Down Expand Up @@ -173,7 +177,9 @@ func parseLineWithToolNames(line []byte, toolCalls map[string]toolCallInfo) (ses
return session.Event{}, false, nil
}
event.Kind = session.EventUserMessage
if classified := classifySkillInjectionByLink(text, raw.IsMeta, raw.SourceToolUseID, toolCalls); classified != nil {
if classified := classifyContinuePrompt(text, raw.IsMeta); classified != nil {
event.User = classified
} else if classified := classifySkillInjectionByLink(text, raw.IsMeta, raw.SourceToolUseID, toolCalls); classified != nil {
event.User = classified
} else if classified := classifyCommandUserMessage(text); classified != nil {
event.User = classified
Expand Down
Loading
Loading