diff --git a/cmd/entire/cli/integration_test/shell_capture_test.go b/cmd/entire/cli/integration_test/shell_capture_test.go new file mode 100644 index 0000000000..af0ddcda42 --- /dev/null +++ b/cmd/entire/cli/integration_test/shell_capture_test.go @@ -0,0 +1,294 @@ +//go:build integration + +package integration + +import ( + "os" + "path/filepath" + "runtime" + "slices" + "testing" +) + +// These tests cover files an agent changes through a shell command, which no +// transcript analyzer can name, so the worktree is the only evidence. + +// TestShellEditToPreExistingUntrackedFile_LinksCommit covers an agent editing, +// through a shell command, a file that was already untracked when the prompt +// started. git status lists it as untracked both before and after, so only the +// size and modification time recorded at prompt start show the turn changed it. +func TestShellEditToPreExistingUntrackedFile_LinksCommit(t *testing.T) { + t.Parallel() + env := NewFeatureBranchEnv(t) + + const ( + editedFile = "scratch/plan.md" + untouchedFile = "scratch/todo.md" + ) + env.WriteFile(editedFile, "# Plan\n") + env.WriteFile(untouchedFile, "# Todo\n") + + sess := env.NewSession() + if err := env.SimulateUserPromptSubmitWithTranscriptPath(sess.ID, sess.TranscriptPath); err != nil { + t.Fatalf("SimulateUserPromptSubmit failed: %v", err) + } + + // The transcript names no file: the edit is a shell command. + sess.CreateTranscript("rewrite scratch/plan.md with sed", nil) + env.WriteFile(editedFile, "# Plan\n\n1. Rewritten by the agent's shell command.\n") + if err := env.SimulateStop(sess.ID, sess.TranscriptPath); err != nil { + t.Fatalf("SimulateStop failed: %v", err) + } + + state, err := env.GetSessionState(sess.ID) + if err != nil || state == nil { + t.Fatalf("GetSessionState failed: %v (state=%v)", err, state) + } + if !containsFile(state.FilesTouched, editedFile) { + t.Errorf("turn end did not record the shell edit to pre-existing untracked %s: FilesTouched=%v", + editedFile, state.FilesTouched) + } + if containsFile(state.FilesTouched, untouchedFile) { + t.Errorf("turn end recorded pre-existing untracked %s, which the turn did not change: FilesTouched=%v", + untouchedFile, state.FilesTouched) + } + + env.GitCommitWithHooks("Add plan", editedFile) + if env.GetCheckpointIDFromCommitMessage(env.GetHeadHash()) == "" { + t.Errorf("commit of the agent's shell edit to %s should carry an Entire-Checkpoint trailer", editedFile) + } +} + +// TestShellRewriteKeepingSizeAndMtime_IsRecorded covers a rewrite that keeps +// the file's length and restores its modification time (as cp -p or touch -r +// would). Only the status-change time shows the edit. +func TestShellRewriteKeepingSizeAndMtime_IsRecorded(t *testing.T) { + t.Parallel() + if runtime.GOOS == "windows" { + t.Skip("Windows exposes no status-change time; size and mtime are all that is compared there") + } + env := NewFeatureBranchEnv(t) + + const editedFile = "scratch/config.ini" + env.WriteFile(editedFile, "mode=aaaa\n") + fullPath := filepath.Join(env.RepoDir, editedFile) + before, err := os.Stat(fullPath) + if err != nil { + t.Fatalf("stat %s: %v", editedFile, err) + } + + sess := env.NewSession() + if err := env.SimulateUserPromptSubmitWithTranscriptPath(sess.ID, sess.TranscriptPath); err != nil { + t.Fatalf("SimulateUserPromptSubmit failed: %v", err) + } + sess.CreateTranscript("switch the mode with a shell command", nil) + env.WriteFile(editedFile, "mode=bbbb\n") + if err := os.Chtimes(fullPath, before.ModTime(), before.ModTime()); err != nil { + t.Fatalf("restore mtime: %v", err) + } + if err := env.SimulateStop(sess.ID, sess.TranscriptPath); err != nil { + t.Fatalf("SimulateStop failed: %v", err) + } + + state, err := env.GetSessionState(sess.ID) + if err != nil || state == nil { + t.Fatalf("GetSessionState failed: %v (state=%v)", err, state) + } + if !containsFile(state.FilesTouched, editedFile) { + t.Errorf("turn end missed a same-size rewrite of %s that restored its mtime: FilesTouched=%v", + editedFile, state.FilesTouched) + } +} + +// TestBackgroundSubagentShellCreatedFile_LinksCommit covers a background +// subagent that creates a file through a shell command after the parent's turn +// has ended. Its transcript names nothing, so only the untracked files that +// appeared since launch show it, and SubagentStop claims those no session has +// recorded. +func TestBackgroundSubagentShellCreatedFile_LinksCommit(t *testing.T) { + t.Parallel() + env := NewFeatureBranchEnv(t) + + const ( + taskToolUseID = "toolu_01ShellCreated" + subagentID = "a6666777788889999" + createdFile = "reports/summary.txt" + ) + + sess := env.NewSession() + sess.CreateTranscript("delegate a background report", nil) + launchBackgroundTask(t, env, sess, taskToolUseID, subagentID) + if err := env.SimulateStop(sess.ID, sess.TranscriptPath); err != nil { + t.Fatalf("SimulateStop failed: %v", err) + } + + env.WriteFile(createdFile, "generated by the subagent's shell command\n") + stopBackgroundTask(t, env, sess, subagentID) + + state, err := env.GetSessionState(sess.ID) + if err != nil || state == nil { + t.Fatalf("GetSessionState failed: %v (state=%v)", err, state) + } + if !containsFile(state.FilesTouched, createdFile) { + t.Errorf("SubagentStop did not record the subagent's shell-created %s: FilesTouched=%v", + createdFile, state.FilesTouched) + } + + env.GitCommitWithHooks("Add report", createdFile) + if env.GetCheckpointIDFromCommitMessage(env.GetHeadHash()) == "" { + t.Errorf("commit of the subagent's shell-created %s should carry an Entire-Checkpoint trailer", createdFile) + } +} + +// TestBackgroundSubagentStop_ClaimsOnlyUnrecordedNewFiles pins which new +// untracked files a background SubagentStop claims for its task. Files that +// existed at launch, files the parent recorded in a later turn, and files +// another session in the worktree recorded are not claimed. A file the user +// created while the parent was idle is claimed along with the subagent's own: +// the worktree cannot tell them apart, and the cost is an extra session link. +func TestBackgroundSubagentStop_ClaimsOnlyUnrecordedNewFiles(t *testing.T) { + t.Parallel() + env := NewFeatureBranchEnv(t) + + const ( + taskToolUseID = "toolu_01ClaimBoundary" + subagentID = "b7777888899990000" + preLaunchFile = "notes/before-launch.md" + parentFile = "docs/parent-turn-two.md" + otherFile = "docs/other-session.md" + userFile = "notes/user-while-idle.md" + subagentFile = "reports/generated.txt" + ) + + env.WriteFile(preLaunchFile, "already here\n") + + parent := env.NewSession() + parent.CreateTranscript("delegate a background report", nil) + launchBackgroundTask(t, env, parent, taskToolUseID, subagentID) + if err := env.SimulateStop(parent.ID, parent.TranscriptPath); err != nil { + t.Fatalf("parent Stop (turn 1) failed: %v", err) + } + + // Parent turn 2 creates a file; its turn end records it. + if err := env.SimulateUserPromptSubmitWithTranscriptPath(parent.ID, parent.TranscriptPath); err != nil { + t.Fatalf("parent prompt (turn 2) failed: %v", err) + } + parent.CreateTranscript("write the parent doc", nil) + env.WriteFile(parentFile, "parent turn two\n") + if err := env.SimulateStop(parent.ID, parent.TranscriptPath); err != nil { + t.Fatalf("parent Stop (turn 2) failed: %v", err) + } + + // Another session in the same worktree records its own file. + other := env.NewSession() + if err := env.SimulateUserPromptSubmitWithTranscriptPath(other.ID, other.TranscriptPath); err != nil { + t.Fatalf("other prompt failed: %v", err) + } + other.CreateTranscript("write the other doc", nil) + env.WriteFile(otherFile, "other session\n") + if err := env.SimulateStop(other.ID, other.TranscriptPath); err != nil { + t.Fatalf("other Stop failed: %v", err) + } + + // While both sessions are idle: the user creates a file, and the subagent's + // shell command creates another. + env.WriteFile(userFile, "written by the user\n") + env.WriteFile(subagentFile, "generated by the subagent's shell command\n") + stopBackgroundTask(t, env, parent, subagentID) + + state, err := env.GetSessionState(parent.ID) + if err != nil || state == nil { + t.Fatalf("GetSessionState failed: %v (state=%v)", err, state) + } + var taskFiles []string + for _, rec := range state.TaskRecords { + if rec.ToolUseID == taskToolUseID { + taskFiles = slices.Sorted(slices.Values(rec.Files)) + } + } + want := []string{userFile, subagentFile} + if !slices.Equal(taskFiles, want) { + t.Errorf("task record files = %v, want %v", taskFiles, want) + } +} + +// TestOverlappingBackgroundTasks_ClaimNoNewFiles covers two background tasks, +// from different sessions, running at the same time and each creating a file +// through a shell command. Nothing tells the two files apart, so neither stop +// claims new untracked files: the first to stop would otherwise take both. +func TestOverlappingBackgroundTasks_ClaimNoNewFiles(t *testing.T) { + t.Parallel() + env := NewFeatureBranchEnv(t) + + const ( + firstToolUseID = "toolu_01OverlapFirst" + firstSubagentID = "c1111222233334444" + firstFile = "reports/first.txt" + secondToolUseID = "toolu_01OverlapSecond" + secondAgentID = "c5555666677778888" + secondFile = "reports/second.txt" + ) + + first := env.NewSession() + first.CreateTranscript("delegate the first report", nil) + launchBackgroundTask(t, env, first, firstToolUseID, firstSubagentID) + if err := env.SimulateStop(first.ID, first.TranscriptPath); err != nil { + t.Fatalf("first Stop failed: %v", err) + } + + second := env.NewSession() + second.CreateTranscript("delegate the second report", nil) + launchBackgroundTask(t, env, second, secondToolUseID, secondAgentID) + if err := env.SimulateStop(second.ID, second.TranscriptPath); err != nil { + t.Fatalf("second Stop failed: %v", err) + } + + env.WriteFile(firstFile, "first subagent's shell output\n") + env.WriteFile(secondFile, "second subagent's shell output\n") + stopBackgroundTask(t, env, first, firstSubagentID) + stopBackgroundTask(t, env, second, secondAgentID) + + for _, sess := range []*Session{first, second} { + state, err := env.GetSessionState(sess.ID) + if err != nil || state == nil { + t.Fatalf("GetSessionState(%s) failed: %v (state=%v)", sess.ID, err, state) + } + if containsFile(state.FilesTouched, firstFile) || containsFile(state.FilesTouched, secondFile) { + t.Errorf("session %s claimed a file that could belong to either overlapping task: FilesTouched=%v", + sess.ID, state.FilesTouched) + } + } +} + +func launchBackgroundTask(t *testing.T, env *TestEnv, sess *Session, toolUseID, subagentID string) { + t.Helper() + if err := env.SimulateUserPromptSubmitWithTranscriptPath(sess.ID, sess.TranscriptPath); err != nil { + t.Fatalf("SimulateUserPromptSubmit failed: %v", err) + } + if err := env.SimulatePreTask(sess.ID, sess.TranscriptPath, toolUseID); err != nil { + t.Fatalf("SimulatePreTask failed: %v", err) + } + if err := env.SimulatePostTask(PostTaskInput{ + SessionID: sess.ID, + TranscriptPath: sess.TranscriptPath, + ToolUseID: toolUseID, + AgentID: subagentID, + Background: true, + }); err != nil { + t.Fatalf("SimulatePostTask (background stub) failed: %v", err) + } +} + +// stopBackgroundTask fires SubagentStop for a subagent whose transcript names +// no file: everything it changed went through shell commands. +func stopBackgroundTask(t *testing.T, env *TestEnv, sess *Session, subagentID string) { + t.Helper() + if err := env.SimulateSubagentStop(SubagentStopInput{ + SessionID: sess.ID, + TranscriptPath: sess.TranscriptPath, + AgentID: subagentID, + AgentTranscriptPath: sess.CreateSubagentTranscript(subagentID, nil), + }); err != nil { + t.Fatalf("SimulateSubagentStop failed: %v", err) + } +} diff --git a/cmd/entire/cli/integration_test/subagent_commit_in_turn_test.go b/cmd/entire/cli/integration_test/subagent_commit_in_turn_test.go index e104ad9684..1349af28a6 100644 --- a/cmd/entire/cli/integration_test/subagent_commit_in_turn_test.go +++ b/cmd/entire/cli/integration_test/subagent_commit_in_turn_test.go @@ -320,6 +320,108 @@ func TestSubagentCheckpoints_CommitAfterBackgroundTaskCompletes_LinksViaFiles(t } } +// TestSubagentCheckpoints_BackgroundTaskCompletesMidTurn_ParentCommitLinks pins +// the ordering from issue #2653: a background subagent appends to a tracked +// file and its completion arrives while the parent is still mid-turn, and the +// parent commits in that same turn, before any Stop. The commit must link to the +// session through the subagent's file and carry the subagent's transcript. +func TestSubagentCheckpoints_BackgroundTaskCompletesMidTurn_ParentCommitLinks(t *testing.T) { + t.Parallel() + env := NewFeatureBranchEnv(t) + + const ( + taskToolUseID = "toolu_01CompletesMidTurn" + subagentID = "f5555666677778888" + editedFile = "notes.txt" + baseContent = "base\n" + editedContent = "base\nadded by the background subagent\n" + ) + env.WriteFile(editedFile, baseContent) + env.GitAdd(editedFile) + env.GitCommit("Add notes") + + sess := env.NewSession() + sess.CreateTranscript("delegate a background task, then commit", nil) + + if err := env.SimulateUserPromptSubmitWithTranscriptPath(sess.ID, sess.TranscriptPath); err != nil { + t.Fatalf("SimulateUserPromptSubmit failed: %v", err) + } + if err := env.SimulatePreTask(sess.ID, sess.TranscriptPath, taskToolUseID); err != nil { + t.Fatalf("SimulatePreTask failed: %v", err) + } + if err := env.SimulatePostTask(PostTaskInput{ + SessionID: sess.ID, + TranscriptPath: sess.TranscriptPath, + ToolUseID: taskToolUseID, + AgentID: subagentID, + Background: true, + }); err != nil { + t.Fatalf("SimulatePostTask (background stub) failed: %v", err) + } + + subagentTranscript := sess.CreateSubagentTranscript(subagentID, []FileChange{ + {Path: editedFile, Content: editedContent}, + }) + env.WriteFile(editedFile, editedContent) + if err := env.SimulateSubagentStop(SubagentStopInput{ + SessionID: sess.ID, + TranscriptPath: sess.TranscriptPath, + AgentID: subagentID, + AgentTranscriptPath: subagentTranscript, + }); err != nil { + t.Fatalf("SimulateSubagentStop failed: %v", err) + } + + state, err := env.GetSessionState(sess.ID) + if err != nil || state == nil { + t.Fatalf("GetSessionState failed: %v (state=%v)", err, state) + } + if !state.Phase.IsActive() || len(state.LiveTaskRecords()) != 0 { + t.Fatalf("precondition: want an ACTIVE parent with only a completed record, got phase=%s records=%+v", + state.Phase, state.TaskRecords) + } + + env.GitCommitWithHooksAsAgent("Update notes", editedFile) + + checkpointID := env.GetCheckpointIDFromCommitMessage(env.GetHeadHash()) + if checkpointID == "" { + t.Fatalf("parent's mid-turn commit of the subagent's file should carry an Entire-Checkpoint trailer") + } + metadataContent, ok := env.ReadFileFromBranch(paths.MetadataBranchName, SessionMetadataPath(checkpointID)) + if !ok { + t.Fatalf("session metadata missing for checkpoint %s", checkpointID) + } + var metadata checkpoint.Metadata + if err := json.Unmarshal([]byte(metadataContent), &metadata); err != nil { + t.Fatalf("parse session metadata: %v", err) + } + if metadata.SessionID != sess.ID { + t.Errorf("checkpoint session = %q, want %q", metadata.SessionID, sess.ID) + } + if !containsFile(metadata.FilesTouched, editedFile) { + t.Errorf("checkpoint files_touched = %v, want it to contain %s", metadata.FilesTouched, editedFile) + } + storedTranscript, ok := env.ReadFileFromBranch(paths.MetadataBranchName, + CheckpointTaskFilePath(checkpointID, taskToolUseID, "agent-"+subagentID+".jsonl")) + if !ok { + t.Fatalf("subagent transcript not materialized under the checkpoint's tasks/ subtree") + } + if !strings.Contains(storedTranscript, editedFile) { + t.Errorf("materialized subagent transcript does not reference %q", editedFile) + } + + if err := env.SimulateStop(sess.ID, sess.TranscriptPath); err != nil { + t.Fatalf("SimulateStop failed: %v", err) + } + state, err = env.GetSessionState(sess.ID) + if err != nil || state == nil { + t.Fatalf("GetSessionState failed: %v (state=%v)", err, state) + } + if containsFile(state.FilesTouched, editedFile) { + t.Errorf("%s still pending after it was committed: %v", editedFile, state.FilesTouched) + } +} + // TestSubagentCheckpoints_JointCommitWithRunningSubagent_KeepsBothSessions pins // that the read-only gate does not drop a co-author. A background subagent is // still running under an IDLE parent, so its edit is not in the parent's diff --git a/cmd/entire/cli/lifecycle.go b/cmd/entire/cli/lifecycle.go index 0afd8c451c..30c80caa20 100644 --- a/cmd/entire/cli/lifecycle.go +++ b/cmd/entire/cli/lifecycle.go @@ -975,6 +975,10 @@ func handleLifecycleTurnEnd(ctx context.Context, ag agent.Agent, event *agent.Ev // Git status catches any tracked file with working-tree changes. relModifiedFiles = mergeUnique(relModifiedFiles, FilterAndNormalizePaths(changes.Modified, repoRoot)) } + // Pre-existing untracked files the turn changed: git status lists them as + // untracked both before and after, so neither New nor Modified carries + // them, and a shell edit leaves no trace in the transcript. + relModifiedFiles = mergeUnique(relModifiedFiles, FilterAndNormalizePaths(preState.ChangedUntrackedFiles(repoRoot), repoRoot)) // Filter detected changes to exclude state already committed to HEAD. // When an agent commits files mid-turn, those changes are condensed by @@ -1669,6 +1673,7 @@ func recordInFlightTaskLaunch(logCtx context.Context, event *agent.Event) error ToolUseID: event.ToolUseID, AgentID: event.SubagentID, StartedAt: time.Now(), + Background: true, SubagentType: event.SubagentType, TaskDescription: event.TaskDescription, } @@ -1811,9 +1816,11 @@ func handleSubagentStopFinal(logCtx context.Context, ag agent.Agent, event *agen // would risk sweeping in the parent's or another agent's later edits. See // subagentCaptureOptions.analyzerFilesOnly. captureErr := completeSubagentTaskRecord(logCtx, ag, event, subagentCaptureOptions{ - bypassNoChangesSkip: true, - analyzerFilesOnly: true, - eventFilesOnly: event.CompletionWithoutLaunch, + bypassNoChangesSkip: true, + analyzerFilesOnly: true, + background: true, + claimUnrecordedNewFiles: !event.CompletionWithoutLaunch, + eventFilesOnly: event.CompletionWithoutLaunch, }) if captureErr != nil { return captureErr @@ -1919,6 +1926,17 @@ type subagentCaptureOptions struct { // its original (correct, worktree-scan-based) behavior unchanged. analyzerFilesOnly bool + // background marks the record as a background task's + // (session.TaskRecord.Background), including one this capture creates + // because no launch marker exists (a resumed run under a new key, or a + // completion without a launch), so overlapping tasks still see it. + background bool + + // claimUnrecordedNewFiles, with analyzerFilesOnly, also captures untracked + // files created since the task launched that no session in the worktree + // has recorded. See unrecordedNewFilesSinceLaunch. + claimUnrecordedNewFiles bool + // eventFilesOnly means the adapter already derived child-scoped files from // a shared parent transcript. Do not resolve or scan a child transcript. eventFilesOnly bool @@ -2059,7 +2077,8 @@ func completeSubagentTaskRecord(logCtx context.Context, ag agent.Agent, event *a // deletions by the subagent are also uncapturable in analyzer-only mode // (only the worktree scan detects them), but those are the lesser // failures: over-capture steals attribution from someone else's work, - // which is worse and harder to notice. + // which is worse and harder to notice. claimUnrecordedNewFiles narrows the + // shell side-effect gap for new files; see unrecordedNewFilesSinceLaunch. var changes *FileChanges if !opts.analyzerFilesOnly { preState, preErr := LoadPreTaskState(logCtx, event.ToolUseID) @@ -2108,6 +2127,9 @@ func completeSubagentTaskRecord(logCtx context.Context, ag agent.Agent, event *a relDeletedFiles = FilterAndNormalizePaths(changes.Deleted, repoRoot) relModifiedFiles = mergeUnique(relModifiedFiles, FilterAndNormalizePaths(changes.Modified, repoRoot)) } + if opts.analyzerFilesOnly && opts.claimUnrecordedNewFiles { + relNewFiles = mergeUnique(relNewFiles, unrecordedNewFilesSinceLaunch(logCtx, event, repoRoot)) + } relModifiedFiles, relNewFiles, relDeletedFiles = strategy.FilterTrackableChanges(logCtx, repoRoot, relModifiedFiles, relNewFiles, relDeletedFiles) // If no changes, skip — unless this is a Final (SubagentStop) capture: a @@ -2140,6 +2162,7 @@ func completeSubagentTaskRecord(logCtx context.Context, ag agent.Agent, event *a ToolUseID: event.ToolUseID, AgentID: event.SubagentID, StartedAt: time.Now(), + Background: opts.background, SubagentType: event.SubagentType, TaskDescription: event.TaskDescription, DeclaredTranscriptPath: subagentTranscriptPath, @@ -2176,6 +2199,62 @@ func completeSubagentTaskRecord(logCtx context.Context, ag agent.Agent, event *a return nil } +// unrecordedNewFilesSinceLaunch returns the untracked files created since a +// background task launched that no session in the worktree has recorded. A +// subagent's shell command can create files its transcript never names, and +// once the parent's next prompt starts they sit in that prompt's untracked +// baseline, so no later turn end claims them either. +// +// Files the parent created during its own turns are excluded: its turn ends +// recorded them. So are files another session has recorded. What remains can +// still include a file the user created while the parent was idle, or one an +// agent mid-turn has not recorded yet; claiming those only adds a session link +// to the commit that carries them, the same as a turn end claiming a user's +// edit made during an active turn. +// +// Returns nil when there is no launch baseline, when the scan cannot tell +// which files other sessions own, or when another background task in the +// worktree ran during this one: a new file could then be either task's, and +// the first to stop would claim both. +func unrecordedNewFilesSinceLaunch(ctx context.Context, event *agent.Event, repoRoot string) []string { + preState, err := LoadPreTaskState(ctx, event.ToolUseID) + if err != nil { + logging.Warn(ctx, "failed to load pre-task state", slog.String("error", err.Error())) + return nil + } + if preState == nil || preState.UntrackedScanSkipped { + return nil + } + changes, err := DetectFileChanges(ctx, preState.PreUntrackedFiles()) + if err != nil { + logStatusDegrade(ctx, "failed to compute files created since task launch", err) + return nil + } + created := FilterAndNormalizePaths(changes.New, repoRoot) + if len(created) == 0 { + return nil + } + claims, ok, err := strategy.LoadBackgroundClaimContext(ctx, event.SessionID, event.ToolUseID) + if err != nil || !ok { + if err != nil { + logging.Warn(ctx, "failed to list files recorded by sessions", slog.String("error", err.Error())) + } + return nil + } + if claims.Overlapped { + logging.Info(ctx, "not claiming new untracked files: another background task overlapped this one", + slog.Int("candidates", len(created))) + return nil + } + unrecorded := make([]string, 0, len(created)) + for _, file := range created { + if _, seen := claims.Recorded[file]; !seen { + unrecorded = append(unrecorded, file) + } + } + return unrecorded +} + // loadLiveTaskRecords returns sessionID's still-in-flight task records, or nil // when there is no state (or it cannot be read — logged, best-effort). func loadLiveTaskRecords(logCtx context.Context, sessionID string) []session.TaskRecord { diff --git a/cmd/entire/cli/session/state.go b/cmd/entire/cli/session/state.go index f0dc6bbc6d..113cd4b60b 100644 --- a/cmd/entire/cli/session/state.go +++ b/cmd/entire/cli/session/state.go @@ -490,6 +490,12 @@ type TaskRecord struct { // StartedAt is when the background launch was observed. StartedAt time.Time `json:"started_at"` + // Background marks a record created at a background launch, which runs + // until a later SubagentStop instead of completing at launch. Its stop + // captures no worktree scan, so another background task overlapping its + // run makes new untracked files ambiguous between the two. + Background bool `json:"background,omitempty"` + // SubagentType and TaskDescription are captured at launch time because // SubagentStop payloads carry no tool_input — a Final-path capture has no // way to derive them from the event itself (ParseSubagentTypeAndDescription diff --git a/cmd/entire/cli/state.go b/cmd/entire/cli/state.go index 5f3da83f57..44d4b3fb8d 100644 --- a/cmd/entire/cli/state.go +++ b/cmd/entire/cli/state.go @@ -9,6 +9,7 @@ import ( "log/slog" "os" "path/filepath" + "slices" "strings" "time" @@ -40,6 +41,14 @@ type PrePromptState struct { // turn. UntrackedScanSkipped bool `json:"untracked_scan_skipped,omitempty"` + // UntrackedFileStats records the size and modification time of each + // regular file in UntrackedFiles. Turn-end compares them to find + // pre-existing untracked files the turn changed: git status reports those + // as untracked both before and after, and an edit made through a shell + // command is never named in the transcript. Absent in state written by + // older versions, which then detect no such changes. + UntrackedFileStats map[string]UntrackedFileStat `json:"untracked_file_stats,omitempty"` + // TranscriptOffset is the unified transcript position when this state was captured. // For Claude Code (JSONL), this is the line count. // For Gemini CLI (JSON), this is the message count. @@ -85,6 +94,108 @@ func (s *PrePromptState) PreUntrackedFiles() []string { return s.UntrackedFiles } +// UntrackedFileStat is enough of an untracked file's metadata to tell whether +// it changed, without hashing its content on every prompt: the stat fields +// git's index compares for the same purpose. Inode and ChangeTime are zero +// where the platform does not expose them (see fileChangeStamp). +type UntrackedFileStat struct { + Size int64 `json:"size"` + ModTime int64 `json:"mod_time_ns"` + Inode uint64 `json:"ino,omitempty"` + ChangeTime int64 `json:"ctime_ns,omitempty"` +} + +// Limits on the prompt-start stat pass, which runs inside UserPromptSubmit and +// so delays every prompt. Above maxUntrackedFileStats files no stats are +// recorded; past untrackedFileStatBudget the pass stops with what it has. +// Either way turn end only loses detection of edits to the files left out, as +// with state written before stats existed. +const ( + maxUntrackedFileStats = 10_000 + untrackedFileStatBudget = 500 * time.Millisecond +) + +// statUntrackedFiles records an UntrackedFileStat for each of files that is a +// regular file in the worktree. Fails open: a file it cannot stat is left out +// and simply never reports a change. +func statUntrackedFiles(ctx context.Context, files []string) map[string]UntrackedFileStat { + if len(files) == 0 { + return nil + } + logCtx := logging.WithComponent(ctx, "state") + if len(files) > maxUntrackedFileStats { + logging.Debug(logCtx, "skipping untracked file stats: too many untracked files", + slog.Int("untracked_files", len(files)), slog.Int("limit", maxUntrackedFileStats)) + return nil + } + repoRoot, err := paths.WorktreeRoot(ctx) + if err != nil { + logging.Debug(logCtx, "skipping untracked file stats: no worktree root", + slog.String("error", err.Error())) + return nil + } + stats, complete := statFilesWithin(repoRoot, files, untrackedFileStatBudget) + if !complete { + logging.Debug(logCtx, "untracked file stats stopped at time budget", + slog.Int("untracked_files", len(files)), slog.Int("recorded", len(stats))) + } + return stats +} + +// statFilesWithin stats files until budget elapses. complete is false when it +// stopped early; the stats gathered so far are still returned. +func statFilesWithin(repoRoot string, files []string, budget time.Duration) (stats map[string]UntrackedFileStat, complete bool) { + deadline := time.Now().Add(budget) + stats = make(map[string]UntrackedFileStat, len(files)) + for _, file := range files { + if time.Now().After(deadline) { + return stats, false + } + if stat, ok := statWorktreeFile(repoRoot, file); ok { + stats[file] = stat + } + } + return stats, true +} + +// ChangedUntrackedFiles returns the files in UntrackedFileStats whose stat +// fields no longer match: pre-existing untracked files changed +// since the prompt started. A file that is gone or no longer a regular file is +// not reported; one that was committed since is reported, and turn-end's +// filterToUncommittedFiles drops it. +func (s *PrePromptState) ChangedUntrackedFiles(repoRoot string) []string { + if s == nil || len(s.UntrackedFileStats) == 0 { + return nil + } + var changed []string + for file, before := range s.UntrackedFileStats { + if now, ok := statWorktreeFile(repoRoot, file); ok && now != before { + changed = append(changed, file) + } + } + slices.Sort(changed) + return changed +} + +// statWorktreeFile stats file, a repo-relative path, through the worktree root +// without following a final symlink. ok is false unless it is a regular file. +func statWorktreeFile(repoRoot, file string) (UntrackedFileStat, bool) { + root, err := worktreedir.OpenAt(repoRoot) + if err != nil { + return UntrackedFileStat{}, false + } + name, err := worktreedir.Name(repoRoot, file) + if err != nil { + return UntrackedFileStat{}, false + } + info, err := root.Lstat(name) + if err != nil || !info.Mode().IsRegular() { + return UntrackedFileStat{}, false + } + inode, changeTime := fileChangeStamp(info) + return UntrackedFileStat{Size: info.Size(), ModTime: info.ModTime().UnixNano(), Inode: inode, ChangeTime: changeTime}, true +} + // normalizePrePromptState migrates deprecated fields after loading from JSON. func (s *PrePromptState) normalizePrePromptState() { // Migrate the oldest field to the intermediate field first @@ -155,6 +266,7 @@ func CapturePrePromptState(ctx context.Context, ag agent.Agent, sessionID, sessi Timestamp: time.Now().UTC().Format(time.RFC3339), UntrackedFiles: untrackedFiles, UntrackedScanSkipped: scanSkipped, + UntrackedFileStats: statUntrackedFiles(ctx, untrackedFiles), TranscriptOffset: transcriptOffset, } diff --git a/cmd/entire/cli/state_test.go b/cmd/entire/cli/state_test.go index 852f8b9490..0a3f00bb90 100644 --- a/cmd/entire/cli/state_test.go +++ b/cmd/entire/cli/state_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/entireio/cli/cmd/entire/cli/agent/claudecode" "github.com/entireio/cli/cmd/entire/cli/paths" @@ -1033,3 +1034,26 @@ func TestFilterToUncommittedFiles_TypechangeToSymlinkIsKept(t *testing.T) { t.Errorf("filterToUncommittedFiles() = %v, want [%s] — a typechange is an uncommitted change", got, relPath) } } + +func TestStatUntrackedFiles_SkipsAboveFileLimit(t *testing.T) { + t.Parallel() + files := make([]string, maxUntrackedFileStats+1) + for i := range files { + files[i] = fmt.Sprintf("build/out-%d.o", i) + } + require.Nil(t, statUntrackedFiles(context.Background(), files)) +} + +func TestStatFilesWithin_StopsAtBudget(t *testing.T) { + t.Parallel() + // A missing root makes every stat fail fast, so only the budget decides + // whether the pass completes. + missingRoot := filepath.Join(t.TempDir(), "missing") + files := []string{"a.txt", "b.txt"} + + _, complete := statFilesWithin(missingRoot, files, -time.Second) + require.False(t, complete) + + _, complete = statFilesWithin(missingRoot, files, time.Minute) + require.True(t, complete) +} diff --git a/cmd/entire/cli/strategy/manual_commit_git.go b/cmd/entire/cli/strategy/manual_commit_git.go index dd6768e32e..4fdc2b9ef1 100644 --- a/cmd/entire/cli/strategy/manual_commit_git.go +++ b/cmd/entire/cli/strategy/manual_commit_git.go @@ -232,6 +232,7 @@ func launchStubTaskRecord(rec session.TaskRecord) session.TaskRecord { ToolUseID: rec.ToolUseID, AgentID: rec.AgentID, StartedAt: rec.StartedAt, + Background: rec.Background, SubagentType: rec.SubagentType, TaskDescription: rec.TaskDescription, TranscriptUnavailable: rec.TranscriptUnavailable, @@ -266,6 +267,9 @@ func applyTaskRecordCompletion(state *SessionState, rec session.TaskRecord) erro if rec.TokenUsageFromTranscript { live.TokenUsageFromTranscript = true } + if rec.Background { + live.Background = true + } if live.AgentID == "" { live.AgentID = rec.AgentID } diff --git a/cmd/entire/cli/strategy/manual_commit_session.go b/cmd/entire/cli/strategy/manual_commit_session.go index 5d899f0a4b..dca8a06b8b 100644 --- a/cmd/entire/cli/strategy/manual_commit_session.go +++ b/cmd/entire/cli/strategy/manual_commit_session.go @@ -206,6 +206,59 @@ func exactWorktreeMatches(states []*SessionState, worktreePath string) []*Sessio return exact } +// BackgroundClaimContext is what a background task's SubagentStop needs to +// decide which new untracked files it may claim. +type BackgroundClaimContext struct { + // Recorded holds every path a session in the task's worktree has in + // FilesTouched. + Recorded map[string]struct{} + + // Overlapped reports that another background task in the worktree was + // running at some point since this task launched: still running, or + // completed after this task's launch. + Overlapped bool +} + +// LoadBackgroundClaimContext loads the BackgroundClaimContext for the task +// toolUseID of sessionID. ok is false when the session has no state, no +// recorded worktree, or no record for the task, so its launch time and the +// files other sessions own are unknown. +func LoadBackgroundClaimContext(ctx context.Context, sessionID, toolUseID string) (claims BackgroundClaimContext, ok bool, err error) { + states, err := ListSessionStates(ctx) + if err != nil { + return BackgroundClaimContext{}, false, err + } + var own *SessionState + for _, state := range states { + if state.SessionID == sessionID { + own = state + break + } + } + if own == nil || own.WorktreePath == "" { + return BackgroundClaimContext{}, false, nil + } + task := own.FindTaskRecord(toolUseID) + if task == nil { + return BackgroundClaimContext{}, false, nil + } + claims.Recorded = make(map[string]struct{}) + for _, state := range exactWorktreeMatches(states, own.WorktreePath) { + for _, file := range state.FilesTouched { + claims.Recorded[file] = struct{}{} + } + for _, rec := range state.TaskRecords { + if !rec.Background || (state.SessionID == sessionID && rec.ToolUseID == toolUseID) { + continue + } + if rec.CompletedAt.IsZero() || rec.CompletedAt.After(task.StartedAt) { + claims.Overlapped = true + } + } + } + return claims, true, nil +} + // findSessionsForWorktree finds all sessions for the given worktree path. // Exact WorktreePath matches win; otherwise sessions recorded in another // worktree of the same repository (shared git common dir) are matched, as long diff --git a/cmd/entire/cli/strategy/touched_file_hashes_test.go b/cmd/entire/cli/strategy/touched_file_hashes_test.go index f524baa158..43b1a62d6c 100644 --- a/cmd/entire/cli/strategy/touched_file_hashes_test.go +++ b/cmd/entire/cli/strategy/touched_file_hashes_test.go @@ -189,6 +189,24 @@ func TestApplyTaskRecordCompletion_DropsStaleTouchedFileHashes(t *testing.T) { assert.Equal(t, map[string]string{"kept.txt": "1111111111111111111111111111111111111111"}, state.TouchedFileHashes) } +// TestTaskRecordCompletion_KeepsBackgroundFlag pins that a background stop +// marks its record Background whether or not a launch marker exists: one it +// creates (a resumed run under a new key) and one launched by an older CLI +// without the flag must both be visible to an overlapping task's claim check. +func TestTaskRecordCompletion_KeepsBackgroundFlag(t *testing.T) { + t.Parallel() + state := &SessionState{} + stop := session.TaskRecord{ToolUseID: "toolu_new", StartedAt: time.Now(), Background: true} + state.AddTaskRecord(launchStubTaskRecord(stop)) + state.AddTaskRecord(session.TaskRecord{ToolUseID: "toolu_old", StartedAt: time.Now()}) + + require.NoError(t, applyTaskRecordCompletion(state, stop)) + require.NoError(t, applyTaskRecordCompletion(state, session.TaskRecord{ToolUseID: "toolu_old", Background: true})) + + assert.True(t, state.FindTaskRecord("toolu_new").Background, "a record created at stop keeps the flag") + assert.True(t, state.FindTaskRecord("toolu_old").Background, "completion sets the flag on an unflagged record") +} + func TestMergeUnhashedFilesTouched_ClearsEmptyMap(t *testing.T) { t.Parallel() state := &SessionState{ diff --git a/cmd/entire/cli/untracked_stat_darwin.go b/cmd/entire/cli/untracked_stat_darwin.go new file mode 100644 index 0000000000..019d2de765 --- /dev/null +++ b/cmd/entire/cli/untracked_stat_darwin.go @@ -0,0 +1,16 @@ +package cli + +import ( + "io/fs" + "syscall" +) + +// fileChangeStamp returns the inode number and status-change time (ctime) of +// info. See the linux implementation. +func fileChangeStamp(info fs.FileInfo) (inode uint64, changeTime int64) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0 + } + return st.Ino, st.Ctimespec.Nano() +} diff --git a/cmd/entire/cli/untracked_stat_linux.go b/cmd/entire/cli/untracked_stat_linux.go new file mode 100644 index 0000000000..4ff316fa15 --- /dev/null +++ b/cmd/entire/cli/untracked_stat_linux.go @@ -0,0 +1,18 @@ +package cli + +import ( + "io/fs" + "syscall" +) + +// fileChangeStamp returns the inode number and status-change time (ctime) of +// info, the fields git's index also compares. ctime moves on every write and +// metadata change and cannot be set from userspace, so a same-size rewrite +// that restores the modification time (cp -p, touch -r) still changes it. +func fileChangeStamp(info fs.FileInfo) (inode uint64, changeTime int64) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0 + } + return st.Ino, st.Ctim.Nano() +} diff --git a/cmd/entire/cli/untracked_stat_other.go b/cmd/entire/cli/untracked_stat_other.go new file mode 100644 index 0000000000..f1a9a47b82 --- /dev/null +++ b/cmd/entire/cli/untracked_stat_other.go @@ -0,0 +1,12 @@ +//go:build !linux && !darwin + +package cli + +import "io/fs" + +// fileChangeStamp has no portable source outside linux and darwin: Windows' +// file attribute data carries no change time. Size and modification time are +// all UntrackedFileStat compares there. +func fileChangeStamp(fs.FileInfo) (inode uint64, changeTime int64) { + return 0, 0 +} diff --git a/docs/architecture/sessions-and-checkpoints.md b/docs/architecture/sessions-and-checkpoints.md index cae94c381a..7ad05b6ae3 100644 --- a/docs/architecture/sessions-and-checkpoints.md +++ b/docs/architecture/sessions-and-checkpoints.md @@ -283,6 +283,7 @@ removed by the purge, not the sweep. See `cmd/entire/cli/session_sweep.go`. A turn end (`SaveStep`) writes no git objects. It records the turn into session state and leaves the transcript on local disk: - `StepCount` is incremented, `FilesTouched` gains the turn's modified, new and deleted paths, and `TouchedFileHashes` records the git blob hash of each modified or new regular file (hashed with `git hash-object`, so clean filters apply exactly as `git add` would) and `""` for each deleted path. A path whose hash is unknown — hashing failed, a symlink, or a path that arrived through a task record, a per-tool hook, or a Codex child-file merge (`MergeUnhashedFilesTouched`, which also drops any hash an earlier step recorded for it) — has no entry and falls back to name matching. Paths this turn named that neither exist nor are recorded deletions are dropped from `FilesTouched` as phantoms. +- The turn's paths come from the transcript analyzer plus `git status` (`DetectFileChanges`): modified and deleted tracked files, and untracked files absent from the prompt-start baseline (`PrePromptState.UntrackedFiles`). An untracked file already in that baseline counts as modified when its stat data no longer matches `PrePromptState.UntrackedFileStats`, since git status reports it as untracked either way and a shell edit leaves no trace in the transcript. The stat data is what git's index compares: size, modification time, inode and status-change time. The change time catches a same-size rewrite that restores the modification time (`cp -p`, `touch -r`); Windows exposes no change time, so only size and modification time are compared there. The stat pass runs inside `UserPromptSubmit`, so it is bounded: above 10,000 untracked files it records no stats, and it stops after 500ms with what it has. Edits to the files it left out are not detected, as with state written before stats existed. - `.entire/metadata//` holds the sanitized transcript copy (`full.jsonl`) and `prompt.txt`. These are local, git-ignored files; nothing is redacted or committed until condensation. ``` @@ -366,7 +367,7 @@ commit has already condensed and removed the record is not captured. - **Workflow agent launch** (subagent-start, non-final): agents a Claude Code Workflow launches have no Agent call of their own — the Workflow's PostToolUse reports a run ID, not agents — so Claude Code's `SubagentStart` hook (installed with matcher `workflow-subagent`; other agent types are ignored at parse time too) records each one's in-flight record, keyed by its `agent_id` as both `ToolUseID` and `AgentID`. The launch is `SubagentLaunchIdempotent`: an existing record for that key is kept rather than replaced, so a repeated `SubagentStart` cannot reset the record while it is still in session state; once condensation has materialized and removed a completed record, a late repeat would start a new live one. Their transcripts live in `subagents/workflows//agent-.jsonl`. The stop's `agent_transcript_path` is used only when it is exactly that file for this session and agent inside the parent's session directory, with no link below it; otherwise it is dropped and the layout lookup applies. That file's run becomes the event's `SubagentRunID`: a resumed run reuses its agents' IDs, so its launch can find the earlier run's completed record and add none, and the stop then completes a record of its own (keyed ``, or `-` while that key is taken) unless a record already holds the same run's transcript, which makes it a repeated stop. The transcript resolvers (`paths.ResolveSubagentTranscriptPath`) probe every run for an agent ID; the session's `subagent_tokens` total counts only the runs the transcript (or, at import, the turn's transcript prefix) launched, named by the Workflow launch's structured `toolUseResult.runId` (`taskType: local_workflow`) or, as a fallback, its result's `Run ID:` lines (`paths.WorkflowRunAgentTranscripts`). A Workflow launched from inside a subagent names its run in that subagent's transcript, not the parent's, so its agents get task records but are not counted in the parent session's `subagent_tokens` — the same limit nested `Agent` subagents have. - **Forked skill launch** (post-task, non-final): a Claude Code skill with `context: fork` runs in an agent that Claude Code reports only in the Skill call's PostToolUse result (`status: "forked"`, `agentId`). Entire installs `PostToolUse[Skill]` → `post-task`, which records the launch under the Skill call's `tool_use_id` with that agent ID; the agent's `SubagentStop` completes it by agent ID. A result with `background: false` completes the record at once with files from the transcript only (`SubagentFilesFromTranscript`: no pre-task baseline exists for a Skill call); a missing `background` is treated as running. The agent can stop again when a background child wakes it; only the first stop updates the record. - **Foreground completion** (post-task, non-final): PostToolUse fires at true completion, so the record is created-and-completed in one step, files and transcript path attached. Claude Code's foreground `SubagentStop` arrives just *before* this PostToolUse, when no record exists yet, and is a no-op. -- **SubagentStop (final, authoritative)**: the real completion signal for background tasks. Claude Code's payload carries `agent_id` but no `tool_use_id`, so `handleSubagentStopFinal` finds the record by `AgentID` (`FindTaskRecordByAgentID`, a live record before a completed one) and adopts its `ToolUseID`, which keys exactly-once completion and the checkpoint's `tasks//` tree. It then completes the live record, bypassing any "no changes, skip" instinct: a read-only subagent (reviewer, search agent) still produced a transcript worth materializing. File attribution is analyzer-only (the subagent's own transcript, never a whole-worktree scan that would sweep in the parent's concurrent work); the accepted trade is that shell side-effect files the transcript never names, and deletions, are under-captured. A record already completed (duplicate/racing Final event) is skipped. Known gap: a subagent continued with `SendMessage` gets a second `SubagentStart`/`SubagentStop` under the same `agent_id` but no new Agent call, so its stop finds the completed record and the resumed run's edits are not attributed to the task. +- **SubagentStop (final, authoritative)**: the real completion signal for background tasks. Claude Code's payload carries `agent_id` but no `tool_use_id`, so `handleSubagentStopFinal` finds the record by `AgentID` (`FindTaskRecordByAgentID`, a live record before a completed one) and adopts its `ToolUseID`, which keys exactly-once completion and the checkpoint's `tasks//` tree. It then completes the live record, bypassing any "no changes, skip" instinct: a read-only subagent (reviewer, search agent) still produced a transcript worth materializing. File attribution comes from the subagent's own transcript, never from a whole-worktree diff that would sweep in the parent's concurrent edits. Untracked files are the exception (`unrecordedNewFilesSinceLaunch`): a file created since launch (absent from the pre-task baseline) that no session in the worktree has recorded is claimed too, so a file a shell command created is not lost. Once the parent's next prompt starts it sits in that prompt's untracked baseline, and no later turn end would claim it. The accepted trade is that a file the user created while the parent was idle is claimed as well, which only adds a session link to the commit that carries it; files the parent's own turns or another session recorded are excluded. The claim is skipped when another background task in the worktree (`TaskRecord.Background`, set at a background launch and by every background stop, including one that creates its record) ran during this one, still live or completed after this task's launch: a new file could then be either task's, and the first to stop would take both. Shell-side modifications of tracked files and deletions are still under-captured here; the parent's next turn end picks up modified tracked files through git status. A record already completed (duplicate/racing Final event) is skipped. Known gap: a subagent continued with `SendMessage` gets a second `SubagentStart`/`SubagentStop` under the same `agent_id` but no new Agent call, so its stop finds the completed record and the resumed run's edits are not attributed to the task. - **SessionEnd sweep** (`completeLiveTaskRecords`): a session closing with tasks still in flight completes every remaining live record, strictly **before** `endSessionNow` marks `PhaseEnded` and eagerly condenses, so the