From 7233e1611a1b0bea3b89ba4c668f221bc493ddfb Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Wed, 14 Oct 2026 13:53:59 -0700 Subject: [PATCH 1/3] Count a turn's step and tokens once when its Stop repeats A retried, duplicate, or concurrent Stop for a file-changing turn ran SaveStep again. Each run added the turn's tokens to the session total and the pending checkpoint window and incremented StepCount, so Cursor (whose checkpoint uses the window) committed double tokens. SaveStep now records which turn its last step came from, in the same locked mutation that counts it. It records the turn's pre-prompt nonce, which TurnStart writes before taking any session lock, so a TurnStart that times out on the lock still gets a fresh one. It also records the transcript size, for Stops whose pre-prompt state is already gone. A repeat merges its files but adds no tokens and no step, unless a commit condensed the window in between. Turns without a TurnStart grow the transcript, so they still count. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4Y330425V53XVKKH7KPKTG9 --- .../integration_test/turn_end_once_test.go | 281 ++++++++++++++++++ cmd/entire/cli/lifecycle.go | 5 +- cmd/entire/cli/session/state.go | 10 + cmd/entire/cli/state.go | 12 + cmd/entire/cli/strategy/manual_commit_git.go | 40 ++- .../cli/strategy/save_step_repeat_test.go | 131 ++++++++ cmd/entire/cli/strategy/strategy.go | 11 + docs/architecture/sessions-and-checkpoints.md | 5 + 8 files changed, 487 insertions(+), 8 deletions(-) create mode 100644 cmd/entire/cli/integration_test/turn_end_once_test.go create mode 100644 cmd/entire/cli/strategy/save_step_repeat_test.go diff --git a/cmd/entire/cli/integration_test/turn_end_once_test.go b/cmd/entire/cli/integration_test/turn_end_once_test.go new file mode 100644 index 0000000000..dd57dd0311 --- /dev/null +++ b/cmd/entire/cli/integration_test/turn_end_once_test.go @@ -0,0 +1,281 @@ +//go:build integration + +package integration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/execx" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A Stop hook can reach Entire more than once for the same turn: a retried +// hook, a duplicate delivery, or two hook processes racing. Each one used to +// run SaveStep again, adding the turn's tokens to the session total and the +// pending checkpoint window a second time and counting a second step. These +// tests drive a file-changing turn followed by a repeated (or concurrent) +// Stop and assert the turn is counted once. + +// writeTurnEndOnceTranscript appends JSONL lines to path, creating it. +func writeTurnEndOnceTranscript(t *testing.T, path string, lines ...string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(t, err) + for _, line := range lines { + if _, err := f.WriteString(line + "\n"); err != nil { + _ = f.Close() + require.NoError(t, err) + } + } + require.NoError(t, f.Close()) +} + +// claudeTurnLines returns one Claude Code turn's transcript lines: the prompt +// and an assistant message carrying the given usage. +func claudeTurnLines(n, input, output int) []string { + return []string{ + fmt.Sprintf(`{"uuid":"user-%d","type":"user","message":{"content":"turn %d"}}`, n, n), + fmt.Sprintf(`{"uuid":"asst-%d","type":"assistant","message":{"id":"msg_%d","content":[{"type":"text","text":"ok"}],"usage":{"input_tokens":%d,"output_tokens":%d}}}`, n, n, input, output), + } +} + +// requireTurnCountedOnce asserts the session's total, its pending checkpoint +// window, and its step count. +func requireTurnCountedOnce(t *testing.T, env *TestEnv, sessionID string, input, output, steps int) { + t.Helper() + state, err := env.GetSessionState(sessionID) + require.NoError(t, err) + require.NotNil(t, state, "session state must exist") + require.NotNil(t, state.TokenUsage, "session total must carry token usage") + require.NotNil(t, state.CheckpointTokenUsage, "pending checkpoint window must carry token usage") + assert.Equal(t, input, state.TokenUsage.InputTokens, "session total input") + assert.Equal(t, output, state.TokenUsage.OutputTokens, "session total output") + assert.Equal(t, input, state.CheckpointTokenUsage.InputTokens, "checkpoint window input") + assert.Equal(t, output, state.CheckpointTokenUsage.OutputTokens, "checkpoint window output") + assert.Equal(t, steps, state.StepCount, "step count") +} + +func TestClaudeCode_RepeatedStopCountsTurnOnce(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + session := env.NewSession() + transcript := session.TranscriptPath + + require.NoError(t, env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "turn 1", transcript)) + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(1, 100, 10)...) + env.WriteFile("turn1.txt", "turn 1\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + require.NoError(t, env.SimulateStop(session.ID, transcript), "repeated Stop") + + requireTurnCountedOnce(t, env, session.ID, 100, 10, 1) + + env.GitCommitWithHooks("Turn 1", "turn1.txt") + cp1 := env.TryGetLatestCheckpointID() + require.NotEmpty(t, cp1) + usage := readCommittedTokenUsage(t, env, cp1) + require.NotNil(t, usage) + require.Equal(t, 100, usage.InputTokens, "checkpoint must hold the turn once") + require.Equal(t, 10, usage.OutputTokens, "checkpoint must hold the turn once") + + // The next turn is a new turn and counts normally. + require.NoError(t, env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "turn 2", transcript)) + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(2, 7, 1)...) + env.WriteFile("turn2.txt", "turn 2\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + state, err := env.GetSessionState(session.ID) + require.NoError(t, err) + require.Equal(t, 107, state.TokenUsage.InputTokens, "session total = both turns once") + require.Equal(t, 7, state.CheckpointTokenUsage.InputTokens, "window = turn 2 only") + require.Equal(t, 1, state.StepCount, "turn 2 is the window's one step") +} + +func TestClaudeCode_ConcurrentStopsCountTurnOnce(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + session := env.NewSession() + transcript := session.TranscriptPath + + require.NoError(t, env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "turn 1", transcript)) + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(1, 100, 10)...) + env.WriteFile("turn1.txt", "turn 1\n") + + runConcurrently(t, 2, func() error { return env.SimulateStop(session.ID, transcript) }) + + requireTurnCountedOnce(t, env, session.ID, 100, 10, 1) +} + +// Without a TurnStart (an agent whose hooks never report prompt submission, +// or the first Stop after Entire was enabled mid-session) every Stop is the +// only evidence of a turn, so each one still records a step. +func TestClaudeCode_StopsWithoutTurnStartStillCountEachTurn(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + session := env.NewSession() + transcript := session.TranscriptPath + + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(1, 100, 10)...) + env.WriteFile("turn1.txt", "turn 1\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + state, err := env.GetSessionState(session.ID) + require.NoError(t, err) + require.NotNil(t, state) + require.Equal(t, 1, state.StepCount, "first Stop after a mid-session enable records its step") + require.NotNil(t, state.TokenUsage) + require.Equal(t, 100, state.TokenUsage.InputTokens) + + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(2, 7, 1)...) + env.WriteFile("turn2.txt", "turn 2\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + state, err = env.GetSessionState(session.ID) + require.NoError(t, err) + require.Equal(t, 2, state.StepCount, "a second turn without TurnStart records its own step") + + // A repeat of that Stop has grown nothing, so it is not a new turn. + require.NoError(t, env.SimulateStop(session.ID, transcript)) + state, err = env.GetSessionState(session.ID) + require.NoError(t, err) + require.Equal(t, 2, state.StepCount, "a repeated Stop without TurnStart is not counted again") +} + +// A turn the agent starts on its own after a prompted turn (a background +// notification, or a prompt hook that never reached Entire) has no TurnStart +// of its own. It must still count, not be mistaken for a repeat of the +// prompted turn's Stop. +func TestClaudeCode_UnpromptedTurnAfterPromptedTurnCounts(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + session := env.NewSession() + transcript := session.TranscriptPath + + require.NoError(t, env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "turn 1", transcript)) + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(1, 100, 10)...) + env.WriteFile("turn1.txt", "turn 1\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(2, 7, 1)...) + env.WriteFile("turn2.txt", "turn 2\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + state, err := env.GetSessionState(session.ID) + require.NoError(t, err) + require.NotNil(t, state) + require.Equal(t, 2, state.StepCount, "the unprompted turn records its own step") + require.NotNil(t, state.TokenUsage) + require.GreaterOrEqual(t, state.TokenUsage.InputTokens, 107, "the unprompted turn's tokens are counted") +} + +func TestCursor_RepeatedStopCountsTurnOnce(t *testing.T) { + t.Parallel() + + for _, concurrent := range []bool{false, true} { + t.Run(fmt.Sprintf("concurrent=%v", concurrent), func(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + env.InitEntireWithAgent(agent.AgentNameCursor) + + cursorProjectDir := t.TempDir() + if resolved, err := filepath.EvalSymlinks(cursorProjectDir); err == nil { + cursorProjectDir = resolved + } + const conversationID = "cursor-turn-end-once" + transcript := filepath.Join(cursorProjectDir, conversationID, conversationID+".jsonl") + writeTurnEndOnceTranscript(t, transcript, + `{"type":"user","text":"add a feature"}`, + `{"type":"assistant","text":"done"}`) + + runCursorHook(t, env, cursorProjectDir, "session-start", map[string]any{ + "conversation_id": conversationID, + "transcript_path": transcript, + }) + runCursorHook(t, env, cursorProjectDir, "before-submit-prompt", map[string]any{ + "conversation_id": conversationID, + "transcript_path": transcript, + "prompt": "add a feature", + }) + env.WriteFile("feature.go", "package main\n") + + stop := map[string]any{ + "conversation_id": conversationID, + "transcript_path": transcript, + "loop_count": 1, + "input_tokens": 5000, + "output_tokens": 50, + "cache_read_tokens": 4000, + "cache_write_tokens": 800, + } + if concurrent { + runConcurrently(t, 2, func() error { + return runCursorHookErr(env, cursorProjectDir, "stop", stop) + }) + } else { + runCursorHook(t, env, cursorProjectDir, "stop", stop) + runCursorHook(t, env, cursorProjectDir, "stop", stop) + } + + // Fresh input = 5000 - 4000 cache read - 800 cache write. + requireTurnCountedOnce(t, env, conversationID, 200, 50, 1) + + env.GitCommitWithHooks("Add feature", "feature.go") + cp := env.TryGetLatestCheckpointID() + require.NotEmpty(t, cp) + usage := readCommittedTokenUsage(t, env, cp) + require.NotNil(t, usage) + require.Equal(t, 200, usage.InputTokens, "checkpoint must hold the turn once") + require.Equal(t, 50, usage.OutputTokens, "checkpoint must hold the turn once") + }) + } +} + +// runConcurrently runs fn n times at once and requires every call succeed. +func runConcurrently(t *testing.T, n int, fn func() error) { + t.Helper() + errs := make([]error, n) + var wg sync.WaitGroup + for i := range n { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = fn() + }() + } + wg.Wait() + for i, err := range errs { + require.NoErrorf(t, err, "concurrent call %d", i) + } +} + +// runCursorHookErr is runCursorHook for goroutines: it reports failure as an +// error instead of failing the test from another goroutine. +func runCursorHookErr(env *TestEnv, cursorProjectDir, hookName string, input map[string]any) error { + inputJSON, err := json.Marshal(input) + if err != nil { + return fmt.Errorf("marshal %s input: %w", hookName, err) + } + cmd := execx.NonInteractive(context.Background(), getTestBinary(), "hooks", "cursor", hookName) + cmd.Dir = env.RepoDir + cmd.Stdin = bytes.NewReader(inputJSON) + cmd.Env = append(env.cliEnv(), "ENTIRE_TEST_CURSOR_PROJECT_DIR="+cursorProjectDir) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("cursor %s hook: %w\noutput: %s", hookName, err, out) + } + return nil +} diff --git a/cmd/entire/cli/lifecycle.go b/cmd/entire/cli/lifecycle.go index 9222b00d13..8b05c48dee 100644 --- a/cmd/entire/cli/lifecycle.go +++ b/cmd/entire/cli/lifecycle.go @@ -1117,11 +1117,12 @@ func handleLifecycleTurnEnd(ctx context.Context, ag agent.Agent, event *agent.Ev } // Get transcript position/identifier from pre-prompt state - var transcriptIdentifierAtStart string + var transcriptIdentifierAtStart, turnNonce string var transcriptLinesAtStart int if preState != nil { transcriptIdentifierAtStart = preState.LastTranscriptIdentifier transcriptLinesAtStart = preState.TranscriptOffset + turnNonce = preState.TurnNonce } // Resolve token usage. Hook-provided counts (e.g., Cursor's stop hook, @@ -1162,6 +1163,8 @@ func handleLifecycleTurnEnd(ctx context.Context, ag agent.Agent, event *agent.Ev StepTranscriptStart: transcriptLinesAtStart, TokenUsage: tokenUsage, SubagentLedgerVersion: codexLedgerVersion, + TurnNonce: turnNonce, + TranscriptSize: len(transcriptData), } if err := strat.SaveStep(ctx, stepCtx); err != nil { diff --git a/cmd/entire/cli/session/state.go b/cmd/entire/cli/session/state.go index cebc669688..8f6b012a03 100644 --- a/cmd/entire/cli/session/state.go +++ b/cmd/entire/cli/session/state.go @@ -215,6 +215,16 @@ type State struct { // JSON tag kept as "checkpoint_count" for backward compatibility with existing state files. StepCount int `json:"checkpoint_count"` + // LastStepTurnNonce and LastStepTranscriptSize identify the turn whose + // Stop recorded the latest step (StepContext.TurnNonce and + // TranscriptSize). SaveStep sets them in the same locked mutation that + // counts the step, and a repeated or concurrent Stop for that turn + // matches them and is not counted again. They are deliberately not keyed + // on TurnID: a TurnStart that times out on the session lock never + // rotates TurnID, but it still writes its pre-prompt nonce. + LastStepTurnNonce string `json:"last_step_turn_nonce,omitempty"` + LastStepTranscriptSize int `json:"last_step_transcript_size,omitempty"` + // CheckpointTranscriptStart is the transcript line offset where the current // checkpoint cycle began. Set to 0 at session start, updated to current // transcript length after each condensation. Used to scope the transcript diff --git a/cmd/entire/cli/state.go b/cmd/entire/cli/state.go index 44d4b3fb8d..abca690487 100644 --- a/cmd/entire/cli/state.go +++ b/cmd/entire/cli/state.go @@ -14,6 +14,7 @@ import ( "time" "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" "github.com/entireio/cli/cmd/entire/cli/entiredir" "github.com/entireio/cli/cmd/entire/cli/gitrepo" "github.com/entireio/cli/cmd/entire/cli/jsonutil" @@ -77,6 +78,14 @@ type PrePromptState struct { // checkpoint-scoped token delta. Empty for agents with transcript-embedded // token data. TokenBaseline json.RawMessage `json:"token_baseline,omitempty"` + + // TurnNonce identifies the turn this state was captured for. Turn-end + // hands it to SaveStep, which records a step at most once per nonce, so a + // repeated or concurrent Stop for the turn is not counted twice. It is + // written here, before any session lock, so a TurnStart that times out on + // the lock still gets a fresh one. Empty in state written by older + // versions. + TurnNonce string `json:"turn_nonce,omitempty"` } // PreUntrackedFiles returns the untracked files list, or nil if the receiver is nil. @@ -269,6 +278,9 @@ func CapturePrePromptState(ctx context.Context, ag agent.Agent, sessionID, sessi UntrackedFileStats: statUntrackedFiles(ctx, untrackedFiles), TranscriptOffset: transcriptOffset, } + if nonce, nonceErr := id.Generate(); nonceErr == nil { + state.TurnNonce = nonce.String() + } // Out-of-band token baseline: agents whose token usage lives outside the // transcript (Antigravity) snapshot their cumulative token position now so diff --git a/cmd/entire/cli/strategy/manual_commit_git.go b/cmd/entire/cli/strategy/manual_commit_git.go index 4fdc2b9ef1..763f3d18f6 100644 --- a/cmd/entire/cli/strategy/manual_commit_git.go +++ b/cmd/entire/cli/strategy/manual_commit_git.go @@ -86,18 +86,30 @@ func (s *ManualCommitStrategy) SaveStep(ctx context.Context, step StepContext) e return ErrMutationSkip } - // LastCheckpointID is intentionally NOT cleared here. It is set during - // condensation and used by handleAmendCommitMsg to restore checkpoint - // trailers on amend operations. - state.StepCount++ + // A Stop can reach us more than once for one turn (a retried or + // duplicate hook, or two racing). The repeat still merges its files, + // which is idempotent, but must not add the turn's tokens or count a + // step again. It does count a step when a commit condensed the window + // in between: its files are then new pending work. + repeat := isRepeatedStep(state, step) + if !repeat || state.StepCount == 0 { + // LastCheckpointID is intentionally NOT cleared here. It is set during + // condensation and used by handleAmendCommitMsg to restore checkpoint + // trailers on amend operations. + state.StepCount++ + if state.StepCount == 1 && !repeat { + state.TranscriptIdentifierAtStart = step.StepTranscriptIdentifier + } + } state.FilesTouched = mergeFilesTouched(state.FilesTouched, step.ModifiedFiles, step.NewFiles, step.DeletedFiles) applyTouchedFileHashes(state, changedFiles, stepFileHashes, recordedDeletions) recordUntrackedDeletions(worktreeRoot, state, untrackedGone) dropPhantomFilesTouched(worktreeRoot, state, changedFiles) - if state.StepCount == 1 { - state.TranscriptIdentifierAtStart = step.StepTranscriptIdentifier + if !repeat { + state.LastStepTurnNonce = step.TurnNonce + state.LastStepTranscriptSize = step.TranscriptSize } - if step.TokenUsage != nil { + if step.TokenUsage != nil && !repeat { state.TokenUsage = accumulateTokenUsage(state.TokenUsage, step.TokenUsage) state.CheckpointTokenUsage = accumulateTokenUsage(state.CheckpointTokenUsage, step.TokenUsage) // step.TokenUsage.SubagentTokens is a cumulative-since-session-start @@ -149,6 +161,7 @@ func (s *ManualCommitStrategy) SaveStep(ctx context.Context, step StepContext) e slog.String("strategy", "manual-commit"), slog.String("checkpoint_type", "session"), slog.Int("checkpoint_count", state.StepCount), + slog.Bool("repeated_stop", repeat), slog.Int("modified_files", len(step.ModifiedFiles)), slog.Int("new_files", len(step.NewFiles)), slog.Int("deleted_files", len(step.DeletedFiles)), @@ -162,6 +175,19 @@ func (s *ManualCommitStrategy) SaveStep(ctx context.Context, step StepContext) e return mutErr } +// isRepeatedStep reports whether step ends the same turn as the step state +// last recorded. A Stop that carries its turn's pre-prompt nonce is matched on +// it, which also covers concurrent Stops: both loaded the nonce before either +// consumed it. A Stop without one (an earlier Stop already consumed it, or the +// turn had no TurnStart) is matched on the transcript instead: a new turn +// grows it, a repeat does not. +func isRepeatedStep(state *SessionState, step StepContext) bool { + if step.TurnNonce != "" { + return state.LastStepTurnNonce == step.TurnNonce + } + return step.TranscriptSize > 0 && state.LastStepTranscriptSize == step.TranscriptSize +} + func invalidateStaleSubagentSnapshot(step *StepContext, state *SessionState) { if step.SubagentLedgerVersion == nil || step.TokenUsage == nil || state.SubagentLedgerVersion == *step.SubagentLedgerVersion { diff --git a/cmd/entire/cli/strategy/save_step_repeat_test.go b/cmd/entire/cli/strategy/save_step_repeat_test.go new file mode 100644 index 0000000000..21a159fec8 --- /dev/null +++ b/cmd/entire/cli/strategy/save_step_repeat_test.go @@ -0,0 +1,131 @@ +package strategy + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/testutil" + + "github.com/stretchr/testify/require" +) + +// saveStepRepeatFixture is a repo with one tracked file and a SaveStep helper +// that records a step for it with the given turn identity. +type saveStepRepeatFixture struct { + t *testing.T + dir string + sessionID string + s *ManualCommitStrategy +} + +func newSaveStepRepeatFixture(t *testing.T) *saveStepRepeatFixture { + t.Helper() + dir := t.TempDir() + testutil.InitRepo(t, dir) + testutil.WriteFile(t, dir, "test.txt", "v0") + testutil.GitAdd(t, dir, "test.txt") + testutil.GitCommit(t, dir, "init") + t.Chdir(dir) + + f := &saveStepRepeatFixture{t: t, dir: dir, sessionID: "save-step-repeat", s: &ManualCommitStrategy{}} + metadataDirAbs := filepath.Join(dir, ".entire", "metadata", f.sessionID) + require.NoError(t, os.MkdirAll(metadataDirAbs, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(metadataDirAbs, paths.TranscriptFileName), + []byte(`{"type":"human","message":{"content":"test"}}`+"\n"), 0o644)) + return f +} + +func (f *saveStepRepeatFixture) save(nonce string, transcriptSize int, identifier string) { + f.t.Helper() + require.NoError(f.t, os.WriteFile(filepath.Join(f.dir, "test.txt"), []byte("changed by "+nonce+identifier), 0o644)) + require.NoError(f.t, f.s.SaveStep(context.Background(), StepContext{ + SessionID: f.sessionID, + MetadataDir: ".entire/metadata/" + f.sessionID, + ModifiedFiles: []string{"test.txt"}, + AgentType: agent.AgentTypeClaudeCode, + StepTranscriptIdentifier: identifier, + TokenUsage: &agent.TokenUsage{InputTokens: 100, OutputTokens: 10}, + TurnNonce: nonce, + TranscriptSize: transcriptSize, + })) +} + +func (f *saveStepRepeatFixture) state() *SessionState { + f.t.Helper() + state, err := f.s.loadSessionState(context.Background(), f.sessionID) + require.NoError(f.t, err) + require.NotNil(f.t, state) + return state +} + +func requireStepTokens(t *testing.T, state *SessionState, steps, sessionInput, windowInput int) { + t.Helper() + require.Equal(t, steps, state.StepCount, "step count") + require.NotNil(t, state.TokenUsage) + require.Equal(t, sessionInput, state.TokenUsage.InputTokens, "session total") + require.NotNil(t, state.CheckpointTokenUsage) + require.Equal(t, windowInput, state.CheckpointTokenUsage.InputTokens, "checkpoint window") +} + +func TestSaveStep_RepeatedTurnNonceCountsOnce(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-a", 500, "uuid-a") + f.save("turn-a", 500, "uuid-a") + requireStepTokens(t, f.state(), 1, 100, 100) +} + +// A TurnStart that times out on the session lock writes its pre-prompt nonce +// but never rotates the session's TurnID. Its turn must still count. +func TestSaveStep_NewTurnNonceCountsEvenWithUnrotatedTurnID(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-a", 500, "uuid-a") + turnID := f.state().TurnID + f.save("turn-b", 900, "uuid-b") + state := f.state() + require.Equal(t, turnID, state.TurnID, "PRECONDITION: TurnID not rotated") + requireStepTokens(t, state, 2, 200, 200) +} + +// A Stop without a pre-prompt nonce (it was consumed by an earlier Stop, or +// the turn had no TurnStart) is a repeat only if the transcript did not grow. +func TestSaveStep_WithoutNonceMatchesOnTranscriptSize(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-a", 500, "uuid-a") + f.save("", 500, "") // repeat whose pre-prompt state is gone + requireStepTokens(t, f.state(), 1, 100, 100) + + f.save("", 900, "") // a new turn without TurnStart grew the transcript + requireStepTokens(t, f.state(), 2, 200, 200) +} + +func TestSaveStep_UnknownTurnIdentityAlwaysCounts(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("", 0, "") + f.save("", 0, "") + requireStepTokens(t, f.state(), 2, 200, 200) +} + +// A commit between a Stop and its repeat condenses the window. The repeat's +// files are then new pending work and need a step, but its tokens were +// already counted and condensed, and the window's start identifier must stay. +func TestSaveStep_RepeatAfterCondensationCountsStepNotTokens(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-a", 500, "uuid-a") + require.NoError(t, MutateSessionState(context.Background(), f.sessionID, func(st *SessionState) error { + st.StepCount = 0 + st.CheckpointTokenUsage = nil + st.TranscriptIdentifierAtStart = "uuid-a" + return nil + })) + + f.save("", 500, "") + state := f.state() + require.Equal(t, 1, state.StepCount, "the repeat's files need a pending step") + require.Equal(t, 100, state.TokenUsage.InputTokens, "the turn's tokens are not re-added") + require.Nil(t, state.CheckpointTokenUsage, "the condensed window gets no tokens back") + require.Equal(t, "uuid-a", state.TranscriptIdentifierAtStart, "a repeat must not reset the window start") +} diff --git a/cmd/entire/cli/strategy/strategy.go b/cmd/entire/cli/strategy/strategy.go index d929b3271a..600f2504bd 100644 --- a/cmd/entire/cli/strategy/strategy.go +++ b/cmd/entire/cli/strategy/strategy.go @@ -140,6 +140,17 @@ type StepContext struct { // while token evidence was extracted. nil means no inventory snapshot; // a pointer to zero is a valid snapshot before the first child is observed. SubagentLedgerVersion *uint64 + + // TurnNonce identifies the turn this step ends: the nonce TurnStart wrote + // into the pre-prompt state. Empty when the Stop has no pre-prompt state + // (a repeated Stop that an earlier one already consumed, or a turn that + // never had a TurnStart). + TurnNonce string + + // TranscriptSize is the byte length of the transcript read at this Stop. + // Without a TurnNonce, an unchanged size is what tells a repeated Stop + // from a new turn. Zero means unknown. + TranscriptSize int } // TaskMetadataDir returns the path to a task's metadata directory diff --git a/docs/architecture/sessions-and-checkpoints.md b/docs/architecture/sessions-and-checkpoints.md index 84d1a91f3d..a66e1ec202 100644 --- a/docs/architecture/sessions-and-checkpoints.md +++ b/docs/architecture/sessions-and-checkpoints.md @@ -292,6 +292,11 @@ A turn end (`SaveStep`) writes no git objects. It records the turn into session └── prompt.txt # Checkpoint-scoped user prompts ``` +**One step per turn.** A Stop can arrive more than once for the same turn: a retried hook, a duplicate delivery, or two racing hook processes. Each repeat reaches `SaveStep` again, so `SaveStep` records which turn its last step came from (`LastStepTurnNonce`, `LastStepTranscriptSize`). It sets them in the same locked mutation that adds tokens and increments `StepCount`. A repeat still merges its files, but adds no tokens and no step. +- A Stop is matched on its turn's `PrePromptState.TurnNonce` when it has one. TurnStart writes the nonce before taking any session lock, so a TurnStart that times out on the lock (and so never rotates `TurnID`) still gets a fresh one. Concurrent Stops both loaded the nonce before either consumed it. +- A Stop without a nonce is matched on the transcript's byte size instead. This covers a repeat whose pre-prompt state an earlier Stop already removed, and a turn that never had a TurnStart. A new turn grows the transcript; a repeat does not. +- If a commit condensed the window between a Stop and its repeat, the repeat still counts a step, because its files are new pending work. It adds no tokens. + Hashing runs before the session lock is taken (`hashTouchedFiles`), and the results are applied inside it (`applyTouchedFileHashes`). Multiple concurrent sessions each keep their own state file and metadata directory, so nothing interleaves. **Pending work.** Whether a session has anything to condense is one predicate, `State.HasPendingWork()` (`session/state.go`): steps, touched files, or task content. Condensation triggers, session-end eager condensation, orphan cleanup, the session sweep, and `entire doctor` all key on it. It ignores `FullyCondensed`; the session sweep, `IsCondensableEndedSession`, and doctor's ENDED classification exclude fully condensed sessions themselves. From ad01f2815e9c55341b0ba53772bc91ca561dd7c5 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Wed, 14 Oct 2026 14:14:52 -0700 Subject: [PATCH 2/3] Require an unchanged transcript to treat a Stop as a repeat The pre-prompt file is per session. A Stop that ran late can read the next turn's nonce, and a nonce match alone would then let it claim that turn and drop the turn's own Stop. A differing nonce still always means a new turn. Co-Authored-By: Claude Opus 5.5 --- cmd/entire/cli/strategy/manual_commit_git.go | 21 ++++++++++++------- .../cli/strategy/save_step_repeat_test.go | 10 +++++++++ cmd/entire/cli/strategy/strategy.go | 4 ++-- docs/architecture/sessions-and-checkpoints.md | 5 +++-- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/cmd/entire/cli/strategy/manual_commit_git.go b/cmd/entire/cli/strategy/manual_commit_git.go index 763f3d18f6..bb29f4ce3d 100644 --- a/cmd/entire/cli/strategy/manual_commit_git.go +++ b/cmd/entire/cli/strategy/manual_commit_git.go @@ -176,16 +176,21 @@ func (s *ManualCommitStrategy) SaveStep(ctx context.Context, step StepContext) e } // isRepeatedStep reports whether step ends the same turn as the step state -// last recorded. A Stop that carries its turn's pre-prompt nonce is matched on -// it, which also covers concurrent Stops: both loaded the nonce before either -// consumed it. A Stop without one (an earlier Stop already consumed it, or the -// turn had no TurnStart) is matched on the transcript instead: a new turn -// grows it, a repeat does not. +// last recorded. A repeat has read the same transcript: a new turn grows it, +// a repeat does not. That covers a Stop without a pre-prompt nonce (an earlier +// Stop already consumed it, or the turn had no TurnStart). A nonce different +// from the recorded one is always a new turn, even over an unchanged +// transcript. But a matching nonce is not enough on its own: the pre-prompt +// file is per session, so a Stop that ran late can read the NEXT turn's +// nonce. The size check keeps it from claiming that turn. func isRepeatedStep(state *SessionState, step StepContext) bool { - if step.TurnNonce != "" { - return state.LastStepTurnNonce == step.TurnNonce + if step.TurnNonce != "" && step.TurnNonce != state.LastStepTurnNonce { + return false } - return step.TranscriptSize > 0 && state.LastStepTranscriptSize == step.TranscriptSize + if step.TranscriptSize > 0 { + return state.LastStepTranscriptSize == step.TranscriptSize + } + return step.TurnNonce != "" } func invalidateStaleSubagentSnapshot(step *StepContext, state *SessionState) { diff --git a/cmd/entire/cli/strategy/save_step_repeat_test.go b/cmd/entire/cli/strategy/save_step_repeat_test.go index 21a159fec8..a8c4d46a72 100644 --- a/cmd/entire/cli/strategy/save_step_repeat_test.go +++ b/cmd/entire/cli/strategy/save_step_repeat_test.go @@ -90,6 +90,16 @@ func TestSaveStep_NewTurnNonceCountsEvenWithUnrotatedTurnID(t *testing.T) { requireStepTokens(t, state, 2, 200, 200) } +// The pre-prompt file is per session, so a slow Stop for one turn can read +// the next turn's nonce. When that turn's own Stop arrives, the transcript +// has grown, so it still counts. +func TestSaveStep_MatchingNonceOverGrownTranscriptCounts(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-b", 500, "uuid-a") // turn A's late Stop, holding turn B's nonce + f.save("turn-b", 900, "uuid-b") // turn B's own Stop + requireStepTokens(t, f.state(), 2, 200, 200) +} + // A Stop without a pre-prompt nonce (it was consumed by an earlier Stop, or // the turn had no TurnStart) is a repeat only if the transcript did not grow. func TestSaveStep_WithoutNonceMatchesOnTranscriptSize(t *testing.T) { diff --git a/cmd/entire/cli/strategy/strategy.go b/cmd/entire/cli/strategy/strategy.go index 600f2504bd..833fc504c0 100644 --- a/cmd/entire/cli/strategy/strategy.go +++ b/cmd/entire/cli/strategy/strategy.go @@ -148,8 +148,8 @@ type StepContext struct { TurnNonce string // TranscriptSize is the byte length of the transcript read at this Stop. - // Without a TurnNonce, an unchanged size is what tells a repeated Stop - // from a new turn. Zero means unknown. + // An unchanged size is what tells a repeated Stop from a new turn (see + // isRepeatedStep). Zero means unknown. TranscriptSize int } diff --git a/docs/architecture/sessions-and-checkpoints.md b/docs/architecture/sessions-and-checkpoints.md index a66e1ec202..216319e61b 100644 --- a/docs/architecture/sessions-and-checkpoints.md +++ b/docs/architecture/sessions-and-checkpoints.md @@ -293,8 +293,9 @@ A turn end (`SaveStep`) writes no git objects. It records the turn into session ``` **One step per turn.** A Stop can arrive more than once for the same turn: a retried hook, a duplicate delivery, or two racing hook processes. Each repeat reaches `SaveStep` again, so `SaveStep` records which turn its last step came from (`LastStepTurnNonce`, `LastStepTranscriptSize`). It sets them in the same locked mutation that adds tokens and increments `StepCount`. A repeat still merges its files, but adds no tokens and no step. -- A Stop is matched on its turn's `PrePromptState.TurnNonce` when it has one. TurnStart writes the nonce before taking any session lock, so a TurnStart that times out on the lock (and so never rotates `TurnID`) still gets a fresh one. Concurrent Stops both loaded the nonce before either consumed it. -- A Stop without a nonce is matched on the transcript's byte size instead. This covers a repeat whose pre-prompt state an earlier Stop already removed, and a turn that never had a TurnStart. A new turn grows the transcript; a repeat does not. +- A repeat has read a transcript of the same byte size: a new turn grows it. This covers a repeat whose pre-prompt state an earlier Stop already removed, and a turn that never had a TurnStart. +- A Stop whose `PrePromptState.TurnNonce` differs from the recorded one is always a new turn. TurnStart writes the nonce before taking any session lock, so a TurnStart that times out on the lock (and so never rotates `TurnID`) still gets a fresh one. +- A matching nonce alone is not enough. The pre-prompt file is per session, so a Stop that ran late can read the next turn's nonce; the size check stops it from claiming that turn. - If a commit condensed the window between a Stop and its repeat, the repeat still counts a step, because its files are new pending work. It adds no tokens. Hashing runs before the session lock is taken (`hashTouchedFiles`), and the results are applied inside it (`applyTouchedFileHashes`). Multiple concurrent sessions each keep their own state file and metadata directory, so nothing interleaves. From 8111bd840d53c5b38a9da14765f13da29ca25453 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Wed, 14 Oct 2026 14:32:06 -0700 Subject: [PATCH 3/3] Document why a new turn nonce wins over an unchanged transcript Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4Y58SKS865VHV7D4MER7SAS --- cmd/entire/cli/strategy/manual_commit_git.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/entire/cli/strategy/manual_commit_git.go b/cmd/entire/cli/strategy/manual_commit_git.go index bb29f4ce3d..4362b77e53 100644 --- a/cmd/entire/cli/strategy/manual_commit_git.go +++ b/cmd/entire/cli/strategy/manual_commit_git.go @@ -183,6 +183,13 @@ func (s *ManualCommitStrategy) SaveStep(ctx context.Context, step StepContext) e // transcript. But a matching nonce is not enough on its own: the pre-prompt // file is per session, so a Stop that ran late can read the NEXT turn's // nonce. The size check keeps it from claiming that turn. +// +// A differing nonce wins over an unchanged size on purpose. A new turn whose +// transcript has not yet grown at its Stop (a slow flush) must not lose its +// tokens. The cost is a TurnStart delivered twice, between two deliveries of +// one Stop, which counts that turn again. Entire already treats such a +// TurnStart as a new turn everywhere else: it rotates TurnID and the +// pre-prompt baseline. func isRepeatedStep(state *SessionState, step StepContext) bool { if step.TurnNonce != "" && step.TurnNonce != state.LastStepTurnNonce { return false