diff --git a/cmd/entire/cli/integration_test/turn_end_once_test.go b/cmd/entire/cli/integration_test/turn_end_once_test.go new file mode 100644 index 0000000000..dd57dd0311 --- /dev/null +++ b/cmd/entire/cli/integration_test/turn_end_once_test.go @@ -0,0 +1,281 @@ +//go:build integration + +package integration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/execx" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A Stop hook can reach Entire more than once for the same turn: a retried +// hook, a duplicate delivery, or two hook processes racing. Each one used to +// run SaveStep again, adding the turn's tokens to the session total and the +// pending checkpoint window a second time and counting a second step. These +// tests drive a file-changing turn followed by a repeated (or concurrent) +// Stop and assert the turn is counted once. + +// writeTurnEndOnceTranscript appends JSONL lines to path, creating it. +func writeTurnEndOnceTranscript(t *testing.T, path string, lines ...string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(t, err) + for _, line := range lines { + if _, err := f.WriteString(line + "\n"); err != nil { + _ = f.Close() + require.NoError(t, err) + } + } + require.NoError(t, f.Close()) +} + +// claudeTurnLines returns one Claude Code turn's transcript lines: the prompt +// and an assistant message carrying the given usage. +func claudeTurnLines(n, input, output int) []string { + return []string{ + fmt.Sprintf(`{"uuid":"user-%d","type":"user","message":{"content":"turn %d"}}`, n, n), + fmt.Sprintf(`{"uuid":"asst-%d","type":"assistant","message":{"id":"msg_%d","content":[{"type":"text","text":"ok"}],"usage":{"input_tokens":%d,"output_tokens":%d}}}`, n, n, input, output), + } +} + +// requireTurnCountedOnce asserts the session's total, its pending checkpoint +// window, and its step count. +func requireTurnCountedOnce(t *testing.T, env *TestEnv, sessionID string, input, output, steps int) { + t.Helper() + state, err := env.GetSessionState(sessionID) + require.NoError(t, err) + require.NotNil(t, state, "session state must exist") + require.NotNil(t, state.TokenUsage, "session total must carry token usage") + require.NotNil(t, state.CheckpointTokenUsage, "pending checkpoint window must carry token usage") + assert.Equal(t, input, state.TokenUsage.InputTokens, "session total input") + assert.Equal(t, output, state.TokenUsage.OutputTokens, "session total output") + assert.Equal(t, input, state.CheckpointTokenUsage.InputTokens, "checkpoint window input") + assert.Equal(t, output, state.CheckpointTokenUsage.OutputTokens, "checkpoint window output") + assert.Equal(t, steps, state.StepCount, "step count") +} + +func TestClaudeCode_RepeatedStopCountsTurnOnce(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + session := env.NewSession() + transcript := session.TranscriptPath + + require.NoError(t, env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "turn 1", transcript)) + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(1, 100, 10)...) + env.WriteFile("turn1.txt", "turn 1\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + require.NoError(t, env.SimulateStop(session.ID, transcript), "repeated Stop") + + requireTurnCountedOnce(t, env, session.ID, 100, 10, 1) + + env.GitCommitWithHooks("Turn 1", "turn1.txt") + cp1 := env.TryGetLatestCheckpointID() + require.NotEmpty(t, cp1) + usage := readCommittedTokenUsage(t, env, cp1) + require.NotNil(t, usage) + require.Equal(t, 100, usage.InputTokens, "checkpoint must hold the turn once") + require.Equal(t, 10, usage.OutputTokens, "checkpoint must hold the turn once") + + // The next turn is a new turn and counts normally. + require.NoError(t, env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "turn 2", transcript)) + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(2, 7, 1)...) + env.WriteFile("turn2.txt", "turn 2\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + state, err := env.GetSessionState(session.ID) + require.NoError(t, err) + require.Equal(t, 107, state.TokenUsage.InputTokens, "session total = both turns once") + require.Equal(t, 7, state.CheckpointTokenUsage.InputTokens, "window = turn 2 only") + require.Equal(t, 1, state.StepCount, "turn 2 is the window's one step") +} + +func TestClaudeCode_ConcurrentStopsCountTurnOnce(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + session := env.NewSession() + transcript := session.TranscriptPath + + require.NoError(t, env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "turn 1", transcript)) + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(1, 100, 10)...) + env.WriteFile("turn1.txt", "turn 1\n") + + runConcurrently(t, 2, func() error { return env.SimulateStop(session.ID, transcript) }) + + requireTurnCountedOnce(t, env, session.ID, 100, 10, 1) +} + +// Without a TurnStart (an agent whose hooks never report prompt submission, +// or the first Stop after Entire was enabled mid-session) every Stop is the +// only evidence of a turn, so each one still records a step. +func TestClaudeCode_StopsWithoutTurnStartStillCountEachTurn(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + session := env.NewSession() + transcript := session.TranscriptPath + + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(1, 100, 10)...) + env.WriteFile("turn1.txt", "turn 1\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + state, err := env.GetSessionState(session.ID) + require.NoError(t, err) + require.NotNil(t, state) + require.Equal(t, 1, state.StepCount, "first Stop after a mid-session enable records its step") + require.NotNil(t, state.TokenUsage) + require.Equal(t, 100, state.TokenUsage.InputTokens) + + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(2, 7, 1)...) + env.WriteFile("turn2.txt", "turn 2\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + state, err = env.GetSessionState(session.ID) + require.NoError(t, err) + require.Equal(t, 2, state.StepCount, "a second turn without TurnStart records its own step") + + // A repeat of that Stop has grown nothing, so it is not a new turn. + require.NoError(t, env.SimulateStop(session.ID, transcript)) + state, err = env.GetSessionState(session.ID) + require.NoError(t, err) + require.Equal(t, 2, state.StepCount, "a repeated Stop without TurnStart is not counted again") +} + +// A turn the agent starts on its own after a prompted turn (a background +// notification, or a prompt hook that never reached Entire) has no TurnStart +// of its own. It must still count, not be mistaken for a repeat of the +// prompted turn's Stop. +func TestClaudeCode_UnpromptedTurnAfterPromptedTurnCounts(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + session := env.NewSession() + transcript := session.TranscriptPath + + require.NoError(t, env.SimulateUserPromptSubmitWithPromptAndTranscriptPath(session.ID, "turn 1", transcript)) + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(1, 100, 10)...) + env.WriteFile("turn1.txt", "turn 1\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + writeTurnEndOnceTranscript(t, transcript, claudeTurnLines(2, 7, 1)...) + env.WriteFile("turn2.txt", "turn 2\n") + require.NoError(t, env.SimulateStop(session.ID, transcript)) + + state, err := env.GetSessionState(session.ID) + require.NoError(t, err) + require.NotNil(t, state) + require.Equal(t, 2, state.StepCount, "the unprompted turn records its own step") + require.NotNil(t, state.TokenUsage) + require.GreaterOrEqual(t, state.TokenUsage.InputTokens, 107, "the unprompted turn's tokens are counted") +} + +func TestCursor_RepeatedStopCountsTurnOnce(t *testing.T) { + t.Parallel() + + for _, concurrent := range []bool{false, true} { + t.Run(fmt.Sprintf("concurrent=%v", concurrent), func(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + env.InitEntireWithAgent(agent.AgentNameCursor) + + cursorProjectDir := t.TempDir() + if resolved, err := filepath.EvalSymlinks(cursorProjectDir); err == nil { + cursorProjectDir = resolved + } + const conversationID = "cursor-turn-end-once" + transcript := filepath.Join(cursorProjectDir, conversationID, conversationID+".jsonl") + writeTurnEndOnceTranscript(t, transcript, + `{"type":"user","text":"add a feature"}`, + `{"type":"assistant","text":"done"}`) + + runCursorHook(t, env, cursorProjectDir, "session-start", map[string]any{ + "conversation_id": conversationID, + "transcript_path": transcript, + }) + runCursorHook(t, env, cursorProjectDir, "before-submit-prompt", map[string]any{ + "conversation_id": conversationID, + "transcript_path": transcript, + "prompt": "add a feature", + }) + env.WriteFile("feature.go", "package main\n") + + stop := map[string]any{ + "conversation_id": conversationID, + "transcript_path": transcript, + "loop_count": 1, + "input_tokens": 5000, + "output_tokens": 50, + "cache_read_tokens": 4000, + "cache_write_tokens": 800, + } + if concurrent { + runConcurrently(t, 2, func() error { + return runCursorHookErr(env, cursorProjectDir, "stop", stop) + }) + } else { + runCursorHook(t, env, cursorProjectDir, "stop", stop) + runCursorHook(t, env, cursorProjectDir, "stop", stop) + } + + // Fresh input = 5000 - 4000 cache read - 800 cache write. + requireTurnCountedOnce(t, env, conversationID, 200, 50, 1) + + env.GitCommitWithHooks("Add feature", "feature.go") + cp := env.TryGetLatestCheckpointID() + require.NotEmpty(t, cp) + usage := readCommittedTokenUsage(t, env, cp) + require.NotNil(t, usage) + require.Equal(t, 200, usage.InputTokens, "checkpoint must hold the turn once") + require.Equal(t, 50, usage.OutputTokens, "checkpoint must hold the turn once") + }) + } +} + +// runConcurrently runs fn n times at once and requires every call succeed. +func runConcurrently(t *testing.T, n int, fn func() error) { + t.Helper() + errs := make([]error, n) + var wg sync.WaitGroup + for i := range n { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = fn() + }() + } + wg.Wait() + for i, err := range errs { + require.NoErrorf(t, err, "concurrent call %d", i) + } +} + +// runCursorHookErr is runCursorHook for goroutines: it reports failure as an +// error instead of failing the test from another goroutine. +func runCursorHookErr(env *TestEnv, cursorProjectDir, hookName string, input map[string]any) error { + inputJSON, err := json.Marshal(input) + if err != nil { + return fmt.Errorf("marshal %s input: %w", hookName, err) + } + cmd := execx.NonInteractive(context.Background(), getTestBinary(), "hooks", "cursor", hookName) + cmd.Dir = env.RepoDir + cmd.Stdin = bytes.NewReader(inputJSON) + cmd.Env = append(env.cliEnv(), "ENTIRE_TEST_CURSOR_PROJECT_DIR="+cursorProjectDir) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("cursor %s hook: %w\noutput: %s", hookName, err, out) + } + return nil +} diff --git a/cmd/entire/cli/lifecycle.go b/cmd/entire/cli/lifecycle.go index 9222b00d13..8b05c48dee 100644 --- a/cmd/entire/cli/lifecycle.go +++ b/cmd/entire/cli/lifecycle.go @@ -1117,11 +1117,12 @@ func handleLifecycleTurnEnd(ctx context.Context, ag agent.Agent, event *agent.Ev } // Get transcript position/identifier from pre-prompt state - var transcriptIdentifierAtStart string + var transcriptIdentifierAtStart, turnNonce string var transcriptLinesAtStart int if preState != nil { transcriptIdentifierAtStart = preState.LastTranscriptIdentifier transcriptLinesAtStart = preState.TranscriptOffset + turnNonce = preState.TurnNonce } // Resolve token usage. Hook-provided counts (e.g., Cursor's stop hook, @@ -1162,6 +1163,8 @@ func handleLifecycleTurnEnd(ctx context.Context, ag agent.Agent, event *agent.Ev StepTranscriptStart: transcriptLinesAtStart, TokenUsage: tokenUsage, SubagentLedgerVersion: codexLedgerVersion, + TurnNonce: turnNonce, + TranscriptSize: len(transcriptData), } if err := strat.SaveStep(ctx, stepCtx); err != nil { diff --git a/cmd/entire/cli/session/state.go b/cmd/entire/cli/session/state.go index cebc669688..8f6b012a03 100644 --- a/cmd/entire/cli/session/state.go +++ b/cmd/entire/cli/session/state.go @@ -215,6 +215,16 @@ type State struct { // JSON tag kept as "checkpoint_count" for backward compatibility with existing state files. StepCount int `json:"checkpoint_count"` + // LastStepTurnNonce and LastStepTranscriptSize identify the turn whose + // Stop recorded the latest step (StepContext.TurnNonce and + // TranscriptSize). SaveStep sets them in the same locked mutation that + // counts the step, and a repeated or concurrent Stop for that turn + // matches them and is not counted again. They are deliberately not keyed + // on TurnID: a TurnStart that times out on the session lock never + // rotates TurnID, but it still writes its pre-prompt nonce. + LastStepTurnNonce string `json:"last_step_turn_nonce,omitempty"` + LastStepTranscriptSize int `json:"last_step_transcript_size,omitempty"` + // CheckpointTranscriptStart is the transcript line offset where the current // checkpoint cycle began. Set to 0 at session start, updated to current // transcript length after each condensation. Used to scope the transcript diff --git a/cmd/entire/cli/state.go b/cmd/entire/cli/state.go index 44d4b3fb8d..abca690487 100644 --- a/cmd/entire/cli/state.go +++ b/cmd/entire/cli/state.go @@ -14,6 +14,7 @@ import ( "time" "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" "github.com/entireio/cli/cmd/entire/cli/entiredir" "github.com/entireio/cli/cmd/entire/cli/gitrepo" "github.com/entireio/cli/cmd/entire/cli/jsonutil" @@ -77,6 +78,14 @@ type PrePromptState struct { // checkpoint-scoped token delta. Empty for agents with transcript-embedded // token data. TokenBaseline json.RawMessage `json:"token_baseline,omitempty"` + + // TurnNonce identifies the turn this state was captured for. Turn-end + // hands it to SaveStep, which records a step at most once per nonce, so a + // repeated or concurrent Stop for the turn is not counted twice. It is + // written here, before any session lock, so a TurnStart that times out on + // the lock still gets a fresh one. Empty in state written by older + // versions. + TurnNonce string `json:"turn_nonce,omitempty"` } // PreUntrackedFiles returns the untracked files list, or nil if the receiver is nil. @@ -269,6 +278,9 @@ func CapturePrePromptState(ctx context.Context, ag agent.Agent, sessionID, sessi UntrackedFileStats: statUntrackedFiles(ctx, untrackedFiles), TranscriptOffset: transcriptOffset, } + if nonce, nonceErr := id.Generate(); nonceErr == nil { + state.TurnNonce = nonce.String() + } // Out-of-band token baseline: agents whose token usage lives outside the // transcript (Antigravity) snapshot their cumulative token position now so diff --git a/cmd/entire/cli/strategy/manual_commit_git.go b/cmd/entire/cli/strategy/manual_commit_git.go index 4fdc2b9ef1..4362b77e53 100644 --- a/cmd/entire/cli/strategy/manual_commit_git.go +++ b/cmd/entire/cli/strategy/manual_commit_git.go @@ -86,18 +86,30 @@ func (s *ManualCommitStrategy) SaveStep(ctx context.Context, step StepContext) e return ErrMutationSkip } - // LastCheckpointID is intentionally NOT cleared here. It is set during - // condensation and used by handleAmendCommitMsg to restore checkpoint - // trailers on amend operations. - state.StepCount++ + // A Stop can reach us more than once for one turn (a retried or + // duplicate hook, or two racing). The repeat still merges its files, + // which is idempotent, but must not add the turn's tokens or count a + // step again. It does count a step when a commit condensed the window + // in between: its files are then new pending work. + repeat := isRepeatedStep(state, step) + if !repeat || state.StepCount == 0 { + // LastCheckpointID is intentionally NOT cleared here. It is set during + // condensation and used by handleAmendCommitMsg to restore checkpoint + // trailers on amend operations. + state.StepCount++ + if state.StepCount == 1 && !repeat { + state.TranscriptIdentifierAtStart = step.StepTranscriptIdentifier + } + } state.FilesTouched = mergeFilesTouched(state.FilesTouched, step.ModifiedFiles, step.NewFiles, step.DeletedFiles) applyTouchedFileHashes(state, changedFiles, stepFileHashes, recordedDeletions) recordUntrackedDeletions(worktreeRoot, state, untrackedGone) dropPhantomFilesTouched(worktreeRoot, state, changedFiles) - if state.StepCount == 1 { - state.TranscriptIdentifierAtStart = step.StepTranscriptIdentifier + if !repeat { + state.LastStepTurnNonce = step.TurnNonce + state.LastStepTranscriptSize = step.TranscriptSize } - if step.TokenUsage != nil { + if step.TokenUsage != nil && !repeat { state.TokenUsage = accumulateTokenUsage(state.TokenUsage, step.TokenUsage) state.CheckpointTokenUsage = accumulateTokenUsage(state.CheckpointTokenUsage, step.TokenUsage) // step.TokenUsage.SubagentTokens is a cumulative-since-session-start @@ -149,6 +161,7 @@ func (s *ManualCommitStrategy) SaveStep(ctx context.Context, step StepContext) e slog.String("strategy", "manual-commit"), slog.String("checkpoint_type", "session"), slog.Int("checkpoint_count", state.StepCount), + slog.Bool("repeated_stop", repeat), slog.Int("modified_files", len(step.ModifiedFiles)), slog.Int("new_files", len(step.NewFiles)), slog.Int("deleted_files", len(step.DeletedFiles)), @@ -162,6 +175,31 @@ func (s *ManualCommitStrategy) SaveStep(ctx context.Context, step StepContext) e return mutErr } +// isRepeatedStep reports whether step ends the same turn as the step state +// last recorded. A repeat has read the same transcript: a new turn grows it, +// a repeat does not. That covers a Stop without a pre-prompt nonce (an earlier +// Stop already consumed it, or the turn had no TurnStart). A nonce different +// from the recorded one is always a new turn, even over an unchanged +// transcript. But a matching nonce is not enough on its own: the pre-prompt +// file is per session, so a Stop that ran late can read the NEXT turn's +// nonce. The size check keeps it from claiming that turn. +// +// A differing nonce wins over an unchanged size on purpose. A new turn whose +// transcript has not yet grown at its Stop (a slow flush) must not lose its +// tokens. The cost is a TurnStart delivered twice, between two deliveries of +// one Stop, which counts that turn again. Entire already treats such a +// TurnStart as a new turn everywhere else: it rotates TurnID and the +// pre-prompt baseline. +func isRepeatedStep(state *SessionState, step StepContext) bool { + if step.TurnNonce != "" && step.TurnNonce != state.LastStepTurnNonce { + return false + } + if step.TranscriptSize > 0 { + return state.LastStepTranscriptSize == step.TranscriptSize + } + return step.TurnNonce != "" +} + func invalidateStaleSubagentSnapshot(step *StepContext, state *SessionState) { if step.SubagentLedgerVersion == nil || step.TokenUsage == nil || state.SubagentLedgerVersion == *step.SubagentLedgerVersion { diff --git a/cmd/entire/cli/strategy/save_step_repeat_test.go b/cmd/entire/cli/strategy/save_step_repeat_test.go new file mode 100644 index 0000000000..a8c4d46a72 --- /dev/null +++ b/cmd/entire/cli/strategy/save_step_repeat_test.go @@ -0,0 +1,141 @@ +package strategy + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/agent" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/testutil" + + "github.com/stretchr/testify/require" +) + +// saveStepRepeatFixture is a repo with one tracked file and a SaveStep helper +// that records a step for it with the given turn identity. +type saveStepRepeatFixture struct { + t *testing.T + dir string + sessionID string + s *ManualCommitStrategy +} + +func newSaveStepRepeatFixture(t *testing.T) *saveStepRepeatFixture { + t.Helper() + dir := t.TempDir() + testutil.InitRepo(t, dir) + testutil.WriteFile(t, dir, "test.txt", "v0") + testutil.GitAdd(t, dir, "test.txt") + testutil.GitCommit(t, dir, "init") + t.Chdir(dir) + + f := &saveStepRepeatFixture{t: t, dir: dir, sessionID: "save-step-repeat", s: &ManualCommitStrategy{}} + metadataDirAbs := filepath.Join(dir, ".entire", "metadata", f.sessionID) + require.NoError(t, os.MkdirAll(metadataDirAbs, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(metadataDirAbs, paths.TranscriptFileName), + []byte(`{"type":"human","message":{"content":"test"}}`+"\n"), 0o644)) + return f +} + +func (f *saveStepRepeatFixture) save(nonce string, transcriptSize int, identifier string) { + f.t.Helper() + require.NoError(f.t, os.WriteFile(filepath.Join(f.dir, "test.txt"), []byte("changed by "+nonce+identifier), 0o644)) + require.NoError(f.t, f.s.SaveStep(context.Background(), StepContext{ + SessionID: f.sessionID, + MetadataDir: ".entire/metadata/" + f.sessionID, + ModifiedFiles: []string{"test.txt"}, + AgentType: agent.AgentTypeClaudeCode, + StepTranscriptIdentifier: identifier, + TokenUsage: &agent.TokenUsage{InputTokens: 100, OutputTokens: 10}, + TurnNonce: nonce, + TranscriptSize: transcriptSize, + })) +} + +func (f *saveStepRepeatFixture) state() *SessionState { + f.t.Helper() + state, err := f.s.loadSessionState(context.Background(), f.sessionID) + require.NoError(f.t, err) + require.NotNil(f.t, state) + return state +} + +func requireStepTokens(t *testing.T, state *SessionState, steps, sessionInput, windowInput int) { + t.Helper() + require.Equal(t, steps, state.StepCount, "step count") + require.NotNil(t, state.TokenUsage) + require.Equal(t, sessionInput, state.TokenUsage.InputTokens, "session total") + require.NotNil(t, state.CheckpointTokenUsage) + require.Equal(t, windowInput, state.CheckpointTokenUsage.InputTokens, "checkpoint window") +} + +func TestSaveStep_RepeatedTurnNonceCountsOnce(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-a", 500, "uuid-a") + f.save("turn-a", 500, "uuid-a") + requireStepTokens(t, f.state(), 1, 100, 100) +} + +// A TurnStart that times out on the session lock writes its pre-prompt nonce +// but never rotates the session's TurnID. Its turn must still count. +func TestSaveStep_NewTurnNonceCountsEvenWithUnrotatedTurnID(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-a", 500, "uuid-a") + turnID := f.state().TurnID + f.save("turn-b", 900, "uuid-b") + state := f.state() + require.Equal(t, turnID, state.TurnID, "PRECONDITION: TurnID not rotated") + requireStepTokens(t, state, 2, 200, 200) +} + +// The pre-prompt file is per session, so a slow Stop for one turn can read +// the next turn's nonce. When that turn's own Stop arrives, the transcript +// has grown, so it still counts. +func TestSaveStep_MatchingNonceOverGrownTranscriptCounts(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-b", 500, "uuid-a") // turn A's late Stop, holding turn B's nonce + f.save("turn-b", 900, "uuid-b") // turn B's own Stop + requireStepTokens(t, f.state(), 2, 200, 200) +} + +// A Stop without a pre-prompt nonce (it was consumed by an earlier Stop, or +// the turn had no TurnStart) is a repeat only if the transcript did not grow. +func TestSaveStep_WithoutNonceMatchesOnTranscriptSize(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-a", 500, "uuid-a") + f.save("", 500, "") // repeat whose pre-prompt state is gone + requireStepTokens(t, f.state(), 1, 100, 100) + + f.save("", 900, "") // a new turn without TurnStart grew the transcript + requireStepTokens(t, f.state(), 2, 200, 200) +} + +func TestSaveStep_UnknownTurnIdentityAlwaysCounts(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("", 0, "") + f.save("", 0, "") + requireStepTokens(t, f.state(), 2, 200, 200) +} + +// A commit between a Stop and its repeat condenses the window. The repeat's +// files are then new pending work and need a step, but its tokens were +// already counted and condensed, and the window's start identifier must stay. +func TestSaveStep_RepeatAfterCondensationCountsStepNotTokens(t *testing.T) { + f := newSaveStepRepeatFixture(t) + f.save("turn-a", 500, "uuid-a") + require.NoError(t, MutateSessionState(context.Background(), f.sessionID, func(st *SessionState) error { + st.StepCount = 0 + st.CheckpointTokenUsage = nil + st.TranscriptIdentifierAtStart = "uuid-a" + return nil + })) + + f.save("", 500, "") + state := f.state() + require.Equal(t, 1, state.StepCount, "the repeat's files need a pending step") + require.Equal(t, 100, state.TokenUsage.InputTokens, "the turn's tokens are not re-added") + require.Nil(t, state.CheckpointTokenUsage, "the condensed window gets no tokens back") + require.Equal(t, "uuid-a", state.TranscriptIdentifierAtStart, "a repeat must not reset the window start") +} diff --git a/cmd/entire/cli/strategy/strategy.go b/cmd/entire/cli/strategy/strategy.go index d929b3271a..833fc504c0 100644 --- a/cmd/entire/cli/strategy/strategy.go +++ b/cmd/entire/cli/strategy/strategy.go @@ -140,6 +140,17 @@ type StepContext struct { // while token evidence was extracted. nil means no inventory snapshot; // a pointer to zero is a valid snapshot before the first child is observed. SubagentLedgerVersion *uint64 + + // TurnNonce identifies the turn this step ends: the nonce TurnStart wrote + // into the pre-prompt state. Empty when the Stop has no pre-prompt state + // (a repeated Stop that an earlier one already consumed, or a turn that + // never had a TurnStart). + TurnNonce string + + // TranscriptSize is the byte length of the transcript read at this Stop. + // An unchanged size is what tells a repeated Stop from a new turn (see + // isRepeatedStep). Zero means unknown. + TranscriptSize int } // TaskMetadataDir returns the path to a task's metadata directory diff --git a/docs/architecture/sessions-and-checkpoints.md b/docs/architecture/sessions-and-checkpoints.md index 84d1a91f3d..216319e61b 100644 --- a/docs/architecture/sessions-and-checkpoints.md +++ b/docs/architecture/sessions-and-checkpoints.md @@ -292,6 +292,12 @@ A turn end (`SaveStep`) writes no git objects. It records the turn into session └── prompt.txt # Checkpoint-scoped user prompts ``` +**One step per turn.** A Stop can arrive more than once for the same turn: a retried hook, a duplicate delivery, or two racing hook processes. Each repeat reaches `SaveStep` again, so `SaveStep` records which turn its last step came from (`LastStepTurnNonce`, `LastStepTranscriptSize`). It sets them in the same locked mutation that adds tokens and increments `StepCount`. A repeat still merges its files, but adds no tokens and no step. +- A repeat has read a transcript of the same byte size: a new turn grows it. This covers a repeat whose pre-prompt state an earlier Stop already removed, and a turn that never had a TurnStart. +- A Stop whose `PrePromptState.TurnNonce` differs from the recorded one is always a new turn. TurnStart writes the nonce before taking any session lock, so a TurnStart that times out on the lock (and so never rotates `TurnID`) still gets a fresh one. +- A matching nonce alone is not enough. The pre-prompt file is per session, so a Stop that ran late can read the next turn's nonce; the size check stops it from claiming that turn. +- If a commit condensed the window between a Stop and its repeat, the repeat still counts a step, because its files are new pending work. It adds no tokens. + Hashing runs before the session lock is taken (`hashTouchedFiles`), and the results are applied inside it (`applyTouchedFileHashes`). Multiple concurrent sessions each keep their own state file and metadata directory, so nothing interleaves. **Pending work.** Whether a session has anything to condense is one predicate, `State.HasPendingWork()` (`session/state.go`): steps, touched files, or task content. Condensation triggers, session-end eager condensation, orphan cleanup, the session sweep, and `entire doctor` all key on it. It ignores `FullyCondensed`; the session sweep, `IsCondensableEndedSession`, and doctor's ENDED classification exclude fully condensed sessions themselves.