lifecycle: keep a no-change turn's tokens for every agent

cmd/entire/cli/integration_test/cursor_token_condensation_test.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
@@ -21,18 +22,30 @@ import (
 
 func runCursorHook(t *testing.T, env *TestEnv, cursorProjectDir, hookName string, input map[string]any) {
 	t.Helper()
+	require.NoError(t, tryCursorHook(t, env, cursorProjectDir, hookName, input))
+}
+
+// tryCursorHook runs a Cursor hook and returns its failure instead of failing
+// the test, so it can run off the test goroutine.
+func tryCursorHook(t *testing.T, env *TestEnv, cursorProjectDir, hookName string, input map[string]any) error {
+	t.Helper()
 
 	inputJSON, err := json.Marshal(input)
-	require.NoError(t, err)
+	if err != nil {
+		return fmt.Errorf("marshal %s input: %w", hookName, err)
+	}
 
 	cmd := execx.NonInteractive(context.Background(), getTestBinary(), "hooks", "cursor", hookName)
 	cmd.Dir = env.RepoDir
 	cmd.Stdin = bytes.NewReader(inputJSON)
 	cmd.Env = append(env.cliEnv(), "ENTIRE_TEST_CURSOR_PROJECT_DIR="+cursorProjectDir)
 
 	out, err := cmd.CombinedOutput()
-	require.NoErrorf(t, err, "cursor %s hook failed\ninput: %s\noutput: %s", hookName, inputJSON, out)
 	t.Logf("cursor %s output: %s", hookName, out)
+	if err != nil {
+		return fmt.Errorf("cursor %s hook failed\ninput: %s\noutput: %s: %w", hookName, inputJSON, out, err)
+	}
+	return nil
 }
 
 func TestCursorTokenUsage_SurvivesCondensation(t *testing.T) {
@@ -203,6 +216,164 @@ func TestCursorTokenUsage_PerCheckpointScoping(t *testing.T) {
 		"checkpoint 2 OutputTokens must be turn 2 only (30), not the cumulative session total (80)")
 }
 
+// TestCursorTokenUsage_TurnWithoutFileChanges covers the checkpoint-less
+// turn-end path: a Stop that changes no files skips SaveStep, so the stop
+// payload's tokens have to be recorded there or they are lost from both the
+// session total and the next checkpoint (Cursor's transcript carries no usage
+// for condensation to recompute from).
+func TestCursorTokenUsage_TurnWithoutFileChanges(t *testing.T) {
+	t.Parallel()
+
+	env := NewFeatureBranchEnv(t)
+	env.InitEntireWithAgent(agent.AgentNameCursor)
+
+	cursorProjectDir := t.TempDir()
+	if resolved, err := filepath.EvalSymlinks(cursorProjectDir); err == nil {
+		cursorProjectDir = resolved
+	}
+
+	const conversationID = "cursor-nochange-session"
+
+	transcriptDir := filepath.Join(cursorProjectDir, conversationID)
+	require.NoError(t, os.MkdirAll(transcriptDir, 0o755))
+	transcriptPath := filepath.Join(transcriptDir, conversationID+".jsonl")
+
+	hook := func(name string, extra map[string]any) {
+		t.Helper()
+		input := map[string]any{"conversation_id": conversationID, "transcript_path": transcriptPath}
+		for k, v := range extra {
+			input[k] = v
+		}
+		runCursorHook(t, env, cursorProjectDir, name, input)
+	}
+	// turn appends a prompt/answer pair, runs before-submit-prompt, writes
+	// files (none for a read-only turn), and stops with the given counts.
+	// Cursor's input_tokens includes cache reads and writes.
+	turn := func(prompt string, files []string, input, output, cacheRead, cacheWrite int) {
+		t.Helper()
+		f, err := os.OpenFile(transcriptPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
+		require.NoError(t, err)
+		_, werr := f.WriteString(`{"type":"user","text":"` + prompt + `"}` + "\n")
+		require.NoError(t, f.Close())
+		require.NoError(t, werr)
+		hook("before-submit-prompt", map[string]any{"prompt": prompt})
+		for _, name := range files {
+			env.WriteFile(name, "package main\n// "+prompt+"\n")
+		}
+		hook("stop", map[string]any{
+			"model":              "cursor-default",
+			"loop_count":         1,
+			"input_tokens":       input,
+			"output_tokens":      output,
+			"cache_read_tokens":  cacheRead,
+			"cache_write_tokens": cacheWrite,
+		})
+	}
+	sessionTokens := func() *agent.TokenUsage {
+		t.Helper()
+		state, err := env.GetSessionState(conversationID)
+		require.NoError(t, err)
+		require.NotNil(t, state, "session state must exist")
+		require.NotNil(t, state.TokenUsage, "session state must carry token usage")
+		return state.TokenUsage
+	}
+
+	hook("session-start", map[string]any{"model": "cursor-default"})
+
+	// Turn 1 writes a file: fresh input 200, output 50, cache 4000/800.
+	turn("turn one", []string{"turn1.go"}, 5000, 50, 4000, 800)
+	// Turn 2 changes nothing: fresh input 100, output 7, cache 300/20.
+	turn("turn two", nil, 420, 7, 300, 20)
+
+	total := sessionTokens()
+	require.Equal(t, 300, total.InputTokens, "session total must include the no-change turn (200+100)")
+	require.Equal(t, 57, total.OutputTokens, "session total must include the no-change turn (50+7)")
+	require.Equal(t, 4300, total.CacheReadTokens)
+	require.Equal(t, 820, total.CacheCreationTokens)
+
+	env.GitCommitWithHooks("Turn 1", "turn1.go")
+	checkpoint1 := env.TryGetLatestCheckpointID()
+	require.NotEmpty(t, checkpoint1, "expected a checkpoint after the commit")
+
+	cp1 := readCommittedTokenUsage(t, env, checkpoint1)
+	require.NotNil(t, cp1, "checkpoint must carry token usage")
+	require.Equal(t, 300, cp1.InputTokens, "checkpoint must hold both turns, the no-change turn once")
+	require.Equal(t, 57, cp1.OutputTokens, "checkpoint must hold both turns, the no-change turn once")
+	require.Equal(t, 4300, cp1.CacheReadTokens)
+	require.Equal(t, 820, cp1.CacheCreationTokens)
+	require.Equal(t, 300, sessionTokens().InputTokens, "condensation must not change the session total")
+
+	// Turn 3 writes a file: the next checkpoint must not repeat turn 2.
+	turn("turn three", []string{"turn3.go"}, 1000, 9, 0, 0)
+	env.GitCommitWithHooks("Turn 3", "turn3.go")
+	checkpoint2 := env.TryGetLatestCheckpointID()
+	require.NotEmpty(t, checkpoint2)
+	require.NotEqual(t, checkpoint1, checkpoint2, "turn 3 must produce a distinct checkpoint")
+
+	cp2 := readCommittedTokenUsage(t, env, checkpoint2)
+	require.NotNil(t, cp2, "checkpoint 2 must carry turn 3 token usage")
+	require.Equal(t, 1000, cp2.InputTokens, "checkpoint 2 must hold turn 3 only")
+	require.Equal(t, 9, cp2.OutputTokens, "checkpoint 2 must hold turn 3 only")
+	require.Equal(t, 1300, sessionTokens().InputTokens, "session total = every turn once")
+}
+
+// TestCursorTokenUsage_RepeatedStopCountsNoChangeTurnOnce pins that a
+// no-change turn's stop-payload tokens are recorded once per turn: Cursor's
+// payload is the turn's own usage, so a repeated or concurrent stop for the
+// same turn would otherwise add it again.
+func TestCursorTokenUsage_RepeatedStopCountsNoChangeTurnOnce(t *testing.T) {
+	t.Parallel()
+
+	env := NewFeatureBranchEnv(t)
+	env.InitEntireWithAgent(agent.AgentNameCursor)
+
+	cursorProjectDir := t.TempDir()
+	if resolved, err := filepath.EvalSymlinks(cursorProjectDir); err == nil {
+		cursorProjectDir = resolved
+	}
+
+	const conversationID = "cursor-repeated-stop-session"
+	transcriptDir := filepath.Join(cursorProjectDir, conversationID)
+	require.NoError(t, os.MkdirAll(transcriptDir, 0o755))
+	transcriptPath := filepath.Join(transcriptDir, conversationID+".jsonl")
+
+	input := func(extra map[string]any) map[string]any {
+		in := map[string]any{"conversation_id": conversationID, "transcript_path": transcriptPath}
+		for k, v := range extra {
+			in[k] = v
+		}
+		return in
+	}
+	startTurn := func(prompt string) {
+		t.Helper()
+		appendLines(t, transcriptPath, `{"type":"user","text":"`+prompt+`"}`)
+		runCursorHook(t, env, cursorProjectDir, "before-submit-prompt", input(map[string]any{"prompt": prompt}))
+	}
+	// Cursor's input_tokens includes cache reads and writes; none here.
+	stop := func(in, out int) func() error {
+		return func() error {
+			return tryCursorHook(t, env, cursorProjectDir, "stop", input(map[string]any{
+				"model": "cursor-default", "loop_count": 1, "input_tokens": in, "output_tokens": out,
+			}))
+		}
+	}
+
+	runCursorHook(t, env, cursorProjectDir, "session-start", input(map[string]any{"model": "cursor-default"}))
+
+	startTurn("turn one")
+	require.NoError(t, stop(100, 10)())
+	requireTurnTokens(t, env, conversationID, 100, 10, "after a no-change turn")
+
+	require.NoError(t, stop(100, 10)(), "a repeated stop must still succeed")
+	requireTurnTokens(t, env, conversationID, 100, 10, "a repeated stop must add nothing")
+
+	startTurn("turn two")
+	for i, err := range runConcurrently(stop(40, 4), stop(40, 4)) {
+		require.NoError(t, err, "concurrent stop %d", i)
+	}
+	requireTurnTokens(t, env, conversationID, 140, 14, "concurrent stops must record the turn once")
+}
+
 func readCommittedTokenUsage(t *testing.T, env *TestEnv, checkpointID string) *agent.TokenUsage {
 	t.Helper()
 	content, found := env.ReadFileFromBranch(paths.MetadataBranchName, SessionMetadataPath(checkpointID))

cmd/entire/cli/integration_test/factoryai_worker_session_test.go

@@ -3,7 +3,12 @@
 package integration
 
 import (
+	"encoding/json"
 	"testing"
+
+	"github.com/entireio/cli/cmd/entire/cli/checkpoint"
+	"github.com/entireio/cli/cmd/entire/cli/paths"
+	"github.com/stretchr/testify/require"
 )
 
 // Factory AI Droid dispatches Workers as detached sessions: the parent's
@@ -128,6 +133,113 @@ func TestFactoryDroidWorkerSessionBecomesTaskCheckpoint(t *testing.T) {
 	})
 }
 
+// TestFactoryDroidReadOnlyWorkerReachesParentCheckpoint covers a Worker turn
+// that changes no files: it must still become a completed task record on the
+// parent, so its transcript and tokens reach the parent's next checkpoint,
+// with the tokens on the task once and never on the Worker's own session.
+func TestFactoryDroidReadOnlyWorkerReachesParentCheckpoint(t *testing.T) {
+	t.Parallel()
+
+	env := NewTestEnv(t)
+	env.InitRepo()
+	env.InitEntire()
+	env.WriteFile(".gitignore", ".entire/\n")
+	env.WriteFile("README.md", "# Test Repository")
+	env.GitAdd(".gitignore")
+	env.GitAdd("README.md")
+	env.GitCommit("Initial commit")
+
+	parent := env.NewFactoryDroidSession()
+	require.NoError(t, env.SimulateFactoryDroidUserPromptSubmit(parent.ID))
+
+	// A read-only Worker: it reads and reports, consuming tokens, and writes
+	// nothing.
+	const toolUseID = "toolu_reader_01"
+	worker := env.NewFactoryDroidSession()
+	require.NoError(t, env.SimulateFactoryDroidUserPromptSubmit(worker.ID))
+	appendLines(t, worker.TranscriptPath,
+		`{"type":"message","id":"w1","message":{"role":"user","content":[{"type":"text","text":"Summarize the repo"}]}}`,
+		`{"type":"message","id":"w2","message":{"role":"assistant","id":"msg_w1","content":[{"type":"text","text":"It is a test repository."}],"usage":{"input_tokens":70,"output_tokens":7}}}`)
+	worker.MarkAsWorkerSession(parent.ID, toolUseID, "worker: Summarize the repo")
+
+	// Twice: a repeated Stop must not count the Worker's usage again.
+	require.NoError(t, env.SimulateFactoryDroidStop(worker.ID, worker.TranscriptPath))
+	require.NoError(t, env.SimulateFactoryDroidStop(worker.ID, worker.TranscriptPath))
+
+	state, err := env.GetSessionState(parent.ID)
+	require.NoError(t, err)
+	require.NotNil(t, state)
+	rec := state.FindTaskRecord(toolUseID)
+	require.NotNil(t, rec, "a read-only Worker turn must still record a task on the parent")
+	require.False(t, rec.CompletedAt.IsZero(), "the record must be completed")
+	require.Empty(t, rec.Files, "a read-only Worker touches no files")
+	require.Equal(t, worker.TranscriptPath, rec.DeclaredTranscriptPath)
+	require.NotNil(t, rec.TokenUsage, "the record must carry the Worker's usage")
+	require.Equal(t, 70, rec.TokenUsage.InputTokens, "the Worker's usage, once")
+	require.Equal(t, 7, rec.TokenUsage.OutputTokens, "the Worker's usage, once")
+
+	if workerState, err := env.GetSessionState(worker.ID); err != nil {
+		t.Fatalf("GetSessionState(worker) failed: %v", err)
+	} else if workerState != nil {
+		require.Zero(t, workerState.StepCount, "a Worker's turn must not step its own session")
+		if workerState.TokenUsage != nil {
+			require.Zero(t, workerState.TokenUsage.InputTokens, "the Worker's usage belongs to the parent's task, not its own session")
+		}
+	}
+
+	// The parent's own turn changes a file; committing it condenses the
+	// parent, Worker task included.
+	env.WriteFile("feature.go", "package main\n")
+	parent.CreateDroidTranscript("Add a feature", []FileChange{{Path: "feature.go", Content: "package main\n"}})
+	require.NoError(t, env.SimulateFactoryDroidStop(parent.ID, parent.TranscriptPath))
+	env.GitCommitWithHooks("Add feature", "feature.go")
+	checkpointID := env.TryGetLatestCheckpointID()
+	require.NotEmpty(t, checkpointID, "expected a checkpoint for the parent's commit")
+
+	raw, ok := env.ReadFileFromBranch(paths.MetadataBranchName, CheckpointTaskFilePath(checkpointID, toolUseID, "task.json"))
+	require.True(t, ok, "the read-only Worker's task must be materialized in the parent's checkpoint")
+	var task checkpoint.TaskRecord
+	require.NoError(t, json.Unmarshal([]byte(raw), &task))
+	require.Equal(t, worker.ID, task.AgentID)
+	require.NotNil(t, task.TokenUsage, "the stored task must carry the Worker's usage: %s", raw)
+	require.Equal(t, 70, task.TokenUsage.InputTokens, "the stored task holds the Worker's usage once: %s", raw)
+	require.Equal(t, 7, task.TokenUsage.OutputTokens, "the stored task holds the Worker's usage once: %s", raw)
+
+	transcript, ok := env.ReadFileFromBranch(paths.MetadataBranchName,
+		CheckpointTaskFilePath(checkpointID, toolUseID, paths.AgentTranscriptFileName(worker.ID)))
+	require.True(t, ok, "the Worker's transcript must be stored with its task")
+	require.Contains(t, transcript, "It is a test repository.")
+
+	if usage := readCommittedTokenUsage(t, env, checkpointID); usage != nil {
+		require.NotEqual(t, 70, usage.InputTokens, "the parent session's own usage must not absorb the Worker's")
+	}
+
+	// A second Worker turn after that commit: the next checkpoint's task must
+	// hold only the second turn, not the first again.
+	require.NoError(t, env.SimulateFactoryDroidUserPromptSubmit(parent.ID))
+	require.NoError(t, env.SimulateFactoryDroidUserPromptSubmitWithTranscript(worker.ID, worker.TranscriptPath))
+	appendLines(t, worker.TranscriptPath,
+		`{"type":"message","id":"w3","message":{"role":"user","content":[{"type":"text","text":"And the tests?"}]}}`,
+		`{"type":"message","id":"w4","message":{"role":"assistant","id":"msg_w2","content":[{"type":"text","text":"There are none."}],"usage":{"input_tokens":30,"output_tokens":3}}}`)
+	require.NoError(t, env.SimulateFactoryDroidStop(worker.ID, worker.TranscriptPath))
+
+	env.WriteFile("feature2.go", "package main\n")
+	parent.CreateDroidTranscript("Add another feature", []FileChange{{Path: "feature2.go", Content: "package main\n"}})
+	require.NoError(t, env.SimulateFactoryDroidStop(parent.ID, parent.TranscriptPath))
+	env.GitCommitWithHooks("Add feature 2", "feature2.go")
+	checkpoint2 := env.TryGetLatestCheckpointID()
+	require.NotEmpty(t, checkpoint2)
+	require.NotEqual(t, checkpointID, checkpoint2, "expected a second checkpoint")
+
+	raw, ok = env.ReadFileFromBranch(paths.MetadataBranchName, CheckpointTaskFilePath(checkpoint2, toolUseID, "task.json"))
+	require.True(t, ok, "the second Worker turn must be materialized in the second checkpoint")
+	var task2 checkpoint.TaskRecord
+	require.NoError(t, json.Unmarshal([]byte(raw), &task2))
+	require.NotNil(t, task2.TokenUsage, "the second stored task must carry the Worker's usage: %s", raw)
+	require.Equal(t, 30, task2.TokenUsage.InputTokens, "the first Worker turn is already in the first checkpoint: %s", raw)
+	require.Equal(t, 3, task2.TokenUsage.OutputTokens, "the first Worker turn is already in the first checkpoint: %s", raw)
+}
+
 // TestFactoryDroidTopLevelSessionStillCheckpoints guards the fallback: an
 // ordinary Droid session has no calling IDs and must keep its own checkpoint.
 func TestFactoryDroidTopLevelSessionStillCheckpoints(t *testing.T) {

cmd/entire/cli/integration_test/hooks.go

@@ -826,10 +826,17 @@ func (r *FactoryDroidHookRunner) runDroidHookInRepoDir(hookName string, inputJSO
 // SimulateUserPromptSubmit simulates the UserPromptSubmit hook for Factory Droid.
 func (r *FactoryDroidHookRunner) SimulateUserPromptSubmit(sessionID string) error {
 	r.T.Helper()
+	return r.SimulateUserPromptSubmitWithTranscript(sessionID, "")
+}
+
+// SimulateUserPromptSubmitWithTranscript simulates UserPromptSubmit with the
+// transcript path Droid reports, so turn start records the transcript offset.
+func (r *FactoryDroidHookRunner) SimulateUserPromptSubmitWithTranscript(sessionID, transcriptPath string) error {
+	r.T.Helper()
 
 	input := map[string]string{
 		"session_id":      sessionID,
-		"transcript_path": "",
+		"transcript_path": transcriptPath,
 		"prompt":          "test prompt",
 	}
 
@@ -999,6 +1006,13 @@ func (env *TestEnv) SimulateFactoryDroidUserPromptSubmit(sessionID string) error
 	return runner.SimulateUserPromptSubmit(sessionID)
 }
 
+// SimulateFactoryDroidUserPromptSubmitWithTranscript is a convenience method on TestEnv.
+func (env *TestEnv) SimulateFactoryDroidUserPromptSubmitWithTranscript(sessionID, transcriptPath string) error {
+	env.T.Helper()
+	runner := NewFactoryDroidHookRunner(env.RepoDir, env.T)
+	return runner.SimulateUserPromptSubmitWithTranscript(sessionID, transcriptPath)
+}
+
 // SimulateFactoryDroidStop is a convenience method on TestEnv.
 func (env *TestEnv) SimulateFactoryDroidStop(sessionID, transcriptPath string) error {
 	env.T.Helper()