From 1277ae843efc0642286fd506b39ebe053bc880f3 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 14:36:57 -0700 Subject: [PATCH 01/16] fix(checkpoint): bound the git-refs pre-push batch push The pre-push hook pushed every queued checkpoint ref in one `git push` with no deadline; only the per-ref retry fallback was bounded. A large backlog, a slow uplink, or an unreachable SSH host could hold the user's `git push` for minutes or indefinitely. - Push queued refs in chunks (checkpointRefPushChunkSize = 25), dequeuing each chunk as it lands so a backlog drains across pushes. - Share one checkpointFlushBudget across the batch and the per-ref fallback in the pre-push path; the explicit migration push stays unbounded. - A failed chunk no longer stops later chunks; only its refs are retried individually. Batching stops after 2 consecutive failed chunks, an SSH auth failure, the budget, or an interrupt. - Stop after one attempt when git cannot connect at all (connect-phase ssh/curl failures, before any chunk landed), and print that line with credentials redacted, instead of retrying ref by ref. - Add `-o ConnectTimeout=30` for OpenSSH under the pre-push hook. - Move the free-text credential redaction helper into gitremote. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4EQ5BPJG653PSJVN6EWAXYK --- cmd/entire/cli/checkpoint/remote/git.go | 96 +++++++- cmd/entire/cli/checkpoint/remote/git_test.go | 110 ++++++++- cmd/entire/cli/gitremote/gitremote.go | 10 + cmd/entire/cli/plugin_gitremote.go | 4 +- cmd/entire/cli/strategy/manual_commit_push.go | 213 ++++++++++------- cmd/entire/cli/strategy/push_common.go | 114 ++++++++- .../cli/strategy/refs_push_bounds_test.go | 216 ++++++++++++++++++ cmd/entire/cli/strategy/refs_push_test.go | 2 +- docs/architecture/ref-checkpoint-backend.md | 11 +- docs/development/checkpoint-implementation.md | 2 +- 10 files changed, 671 insertions(+), 107 deletions(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index 2b4a9914c0..da7756e842 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -14,7 +14,9 @@ import ( "strings" "sync" "time" + "unicode" + "github.com/entireio/cli/cmd/entire/cli/gitremote" "github.com/entireio/cli/cmd/entire/cli/logging" "github.com/entireio/cli/cmd/entire/cli/settings" ) @@ -104,6 +106,71 @@ func LooksLikeSSHAuthFailure(errText string) bool { return false } +// unreachableRemoteNeedles are connect-phase failures printed by ssh and curl +// themselves, so they are not translated by git's locale. Deliberately absent: +// bare strerror text ("Connection refused", "Connection timed out") — curl's +// trailing strerror is localized, and a timeout or reset mid-upload means the +// host was reachable — and git's own "Could not read from remote repository" +// epilogue, which is translated and also follows auth failures. +var unreachableRemoteNeedles = []string{ + "ssh: connect to host ", + "ssh: could not resolve hostname", + "could not resolve host:", + "could not resolve proxy:", + "failed to connect to ", + "couldn't connect to server", + "resolving timed out", +} + +// maxUnreachableLineRunes caps the cause line printed inside the user's push. +const maxUnreachableLineRunes = 200 + +// UnreachableRemoteLine reports whether err shows the remote could not be +// reached at all (name resolution or TCP connect failed), and returns the line +// that said so, credentials redacted and capped, for display. Lines relayed +// from the remote ("remote: ...") and push porcelain are skipped: a remote that +// printed them was reachable, even when its own hook failed to resolve a host. +func UnreachableRemoteLine(err error) (string, bool) { + if err == nil { + return "", false + } + text := err.Error() + var pushErr *PushError + if errors.As(err, &pushErr) { + text = pushErr.Output() // Line breaks preserved, unlike Error(). + } + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "remote:") || strings.HasPrefix(line, "!") || strings.HasPrefix(line, "=") { + continue + } + lower := strings.ToLower(line) + for _, n := range unreachableRemoteNeedles { + if strings.Contains(lower, n) { + return displayGitLine(line), true + } + } + } + return "", false +} + +// displayGitLine makes one line of git output safe to print: no "fatal: " +// prefix, no URL credentials, no control bytes, at most maxUnreachableLineRunes. +func displayGitLine(line string) string { + line = strings.TrimPrefix(line, "fatal: ") + line = gitremote.RedactCredentialsInText(line) + line = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return -1 + } + return r + }, line) + if r := []rune(line); len(r) > maxUnreachableLineRunes { + line = string(r[:maxUnreachableLineRunes]) + "…" + } + return line +} + // batchModeOptionRe matches an explicit BatchMode ssh option (e.g. // "-o BatchMode=yes" or "BatchMode=no"), case-insensitively. Anchored with \b // so it doesn't false-positive on unrelated text that merely contains @@ -117,6 +184,28 @@ func hasExplicitBatchMode(sshCmd string) bool { return batchModeOptionRe.MatchString(sshCmd) } +// connectTimeoutOptionRe matches an explicit ConnectTimeout ssh option, in the +// same forms batchModeOptionRe accepts. +var connectTimeoutOptionRe = regexp.MustCompile(`(?i)\bConnectTimeout\s*=\s*\S+`) + +// nonInteractiveSSHConnectTimeout bounds the TCP connect to an unreachable host, +// whose kernel default (~75-130s) would otherwise spend most of the pre-push +// checkpoint budget on one attempt. Generous for any reachable host. +const nonInteractiveSSHConnectTimeout = "30" + +// isOpenSSHCommand reports whether sshCmd runs OpenSSH, by the basename of its +// first word — the same guess git makes for its ssh variant. Options are only +// added for OpenSSH: other clients (plink, custom wrappers) may reject -o. +func isOpenSSHCommand(sshCmd string) bool { + fields := strings.Fields(sshCmd) + if len(fields) == 0 { + return false + } + prog := strings.Trim(fields[0], `"'`) + base := strings.ToLower(prog[strings.LastIndexAny(prog, `/\`)+1:]) + return base == "ssh" || base == "ssh.exe" +} + // envLookup returns the value of the last occurrence of key in env (matching // exec.Cmd's last-wins semantics for duplicate entries) and whether it was // found. @@ -165,7 +254,7 @@ func effectiveSSHCommand(ctx context.Context, env []string) string { } // withBatchModeSSH returns env with GIT_SSH_COMMAND set so ssh runs with -// BatchMode=yes. The base ssh invocation is resolved via effectiveSSHCommand +// BatchMode=yes and, for OpenSSH, a ConnectTimeout. The base ssh invocation is resolved via effectiveSSHCommand // (env GIT_SSH_COMMAND > core.sshCommand > GIT_SSH > plain "ssh") so a custom // ssh command configured via core.sshCommand isn't silently discarded. The // flag is only appended when BatchMode isn't already explicitly set — an @@ -184,6 +273,11 @@ func withBatchModeSSH(ctx context.Context, env []string) []string { if !hasExplicitBatchMode(base) { base += " -o BatchMode=yes" } + // Command-line -o beats ~/.ssh/config, so a ConnectTimeout set there is + // overridden here; one set on the command itself is left alone. + if isOpenSSHCommand(base) && !connectTimeoutOptionRe.MatchString(base) { + base += " -o ConnectTimeout=" + nonInteractiveSSHConnectTimeout + } return append(out, key+base) } diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index 89353a0ba1..6014efc2c6 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -1119,35 +1119,35 @@ func TestWithBatchModeSSH(t *testing.T) { { name: "no existing GIT_SSH_COMMAND or config defaults to ssh", in: func(t *testing.T) []string { return isolatedSSHEnv(t) }, - want: "ssh -o BatchMode=yes", + want: "ssh -o BatchMode=yes -o ConnectTimeout=30", }, { name: "preserves and extends a custom ssh command", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=ssh -i /home/me/.ssh/id") }, - want: "ssh -i /home/me/.ssh/id -o BatchMode=yes", + want: "ssh -i /home/me/.ssh/id -o BatchMode=yes -o ConnectTimeout=30", }, { name: "GIT_SSH_COMMAND with explicit BatchMode=yes is left untouched", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=ssh -o BatchMode=yes") }, - want: "ssh -o BatchMode=yes", + want: "ssh -o BatchMode=yes -o ConnectTimeout=30", }, { name: "GIT_SSH_COMMAND with explicit BatchMode=no is respected, not overridden", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=ssh -o BatchMode=no") }, - want: "ssh -o BatchMode=no", + want: "ssh -o BatchMode=no -o ConnectTimeout=30", }, { name: "blank GIT_SSH_COMMAND falls back to ssh", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND= ") }, - want: "ssh -o BatchMode=yes", + want: "ssh -o BatchMode=yes -o ConnectTimeout=30", }, { name: "core.sshCommand git config is used as the base when env is unset", @@ -1155,7 +1155,7 @@ func TestWithBatchModeSSH(t *testing.T) { cfg := gitConfigFile(t, "ssh -i /home/me/.ssh/work_key") return isolatedSSHEnv(t, cfg) }, - want: "ssh -i /home/me/.ssh/work_key -o BatchMode=yes", + want: "ssh -i /home/me/.ssh/work_key -o BatchMode=yes -o ConnectTimeout=30", }, { name: "GIT_SSH_COMMAND env takes precedence over core.sshCommand config", @@ -1163,7 +1163,7 @@ func TestWithBatchModeSSH(t *testing.T) { cfg := gitConfigFile(t, "ssh -i /home/me/.ssh/work_key") return isolatedSSHEnv(t, cfg, "GIT_SSH_COMMAND=ssh -i /home/me/.ssh/personal_key") }, - want: "ssh -i /home/me/.ssh/personal_key -o BatchMode=yes", + want: "ssh -i /home/me/.ssh/personal_key -o BatchMode=yes -o ConnectTimeout=30", }, { name: "GIT_SSH is used only when neither env GIT_SSH_COMMAND nor config are set", @@ -1177,7 +1177,28 @@ func TestWithBatchModeSSH(t *testing.T) { in: func(t *testing.T) []string { return isolatedSSHEnv(t, `GIT_SSH_COMMAND=ssh -o ProxyCommand="connect -H proxy NoBatchModeHereEither"`) }, - want: `ssh -o ProxyCommand="connect -H proxy NoBatchModeHereEither" -o BatchMode=yes`, + want: `ssh -o ProxyCommand="connect -H proxy NoBatchModeHereEither" -o BatchMode=yes -o ConnectTimeout=30`, + }, + { + name: "explicit ConnectTimeout is left untouched", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=ssh -o ConnectTimeout=5") + }, + want: "ssh -o ConnectTimeout=5 -o BatchMode=yes", + }, + { + name: "absolute OpenSSH path gets ConnectTimeout", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=/usr/bin/ssh -i key") + }, + want: "/usr/bin/ssh -i key -o BatchMode=yes -o ConnectTimeout=30", + }, + { + name: "non-OpenSSH client gets no ConnectTimeout", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=plink -batch") + }, + want: "plink -batch -o BatchMode=yes", }, } @@ -1192,6 +1213,77 @@ func TestWithBatchModeSSH(t *testing.T) { } } +func TestUnreachableRemoteLine(t *testing.T) { + t.Parallel() + pushErr := func(output string) error { + return fmt.Errorf("push 2 checkpoint refs: %w", &PushError{cause: errors.New("exit status 128"), detail: output, output: output}) + } + tests := []struct { + name string + err error + want string + ok bool + }{ + { + name: "ssh connect refused", + err: pushErr("ssh: connect to host github.com port 22: Connection refused\n" + + "fatal: Could not read from remote repository."), + want: "ssh: connect to host github.com port 22: Connection refused", + ok: true, + }, + { + name: "ssh unresolvable host", + err: pushErr("ssh: Could not resolve hostname github.invalid: nodename nor servname provided"), + want: "ssh: Could not resolve hostname github.invalid: nodename nor servname provided", + ok: true, + }, + { + name: "https unresolvable host, credentials redacted", + err: pushErr("fatal: unable to access 'https://bob:hunter2@git.example.com/x.git/': Could not resolve host: git.example.com"), + want: "unable to access 'https://git.example.com/x.git/': Could not resolve host: git.example.com", + ok: true, + }, + { + name: "https connect failure", + err: pushErr("fatal: unable to access 'https://github.com/o/r.git/': Failed to connect to github.com port 443 after 75003 ms: Couldn't connect to server"), + want: "unable to access 'https://github.com/o/r.git/': Failed to connect to github.com port 443 after 75003 ms: Couldn't connect to server", + ok: true, + }, + { + name: "plain error without PushError", + err: errors.New("git push: exit status 128 (ssh: connect to host h port 22: Operation timed out)"), + want: "git push: exit status 128 (ssh: connect to host h port 22: Operation timed out)", + ok: true, + }, + { + name: "remote's own hook failing to resolve a host means the remote was reached", + err: pushErr("remote: curl: (6) Could not resolve host: ci.internal\n ! [remote rejected] refs/x -> refs/x (pre-receive hook declined)"), + }, + { + name: "mid-upload curl timeout means the remote was reached", + err: pushErr("error: RPC failed; curl 28 Operation timed out after 300000 milliseconds with 0 bytes received\nsend-pack: unexpected disconnect while reading sideband packet"), + }, + { + name: "auth failure is not unreachability", + err: pushErr("git@github.com: Permission denied (publickey).\n" + + "fatal: Could not read from remote repository."), + }, + { + name: "non-fast-forward rejection", + err: pushErr(" ! [rejected] refs/x -> refs/x (non-fast-forward)\nerror: failed to push some refs"), + }, + {name: "nil", err: nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, ok := UnreachableRemoteLine(tt.err) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.want, got) + }) + } +} + func TestWithBatchModeSSH_PreservesOtherVarsWithoutDuplicating(t *testing.T) { t.Parallel() @@ -1209,7 +1301,7 @@ func TestWithBatchModeSSH_PreservesOtherVarsWithoutDuplicating(t *testing.T) { m := envToMap(out) assert.Equal(t, "value", m["SOME_OTHER_VAR"]) - assert.Equal(t, "ssh -o BatchMode=yes", m["GIT_SSH_COMMAND"]) + assert.Equal(t, "ssh -o BatchMode=yes -o ConnectTimeout=30", m["GIT_SSH_COMMAND"]) } // TestNewCommand_NonInteractiveSSH verifies that a checkpoint git command built diff --git a/cmd/entire/cli/gitremote/gitremote.go b/cmd/entire/cli/gitremote/gitremote.go index 525198f760..1a7164a3a8 100644 --- a/cmd/entire/cli/gitremote/gitremote.go +++ b/cmd/entire/cli/gitremote/gitremote.go @@ -9,6 +9,7 @@ import ( "fmt" "net/url" "os/exec" + "regexp" "strings" "unicode" ) @@ -383,6 +384,15 @@ func RedactURL(rawURL string) string { return u.Scheme + "://" + u.Host + u.Path } +// credentialInURL matches the scheme and userinfo of a URL embedded in text. +var credentialInURL = regexp.MustCompile(`([a-zA-Z][a-zA-Z0-9+.-]*://)[^/@\s]*@`) + +// RedactCredentialsInText removes userinfo from any URL appearing in free text, +// for git's output where we do not know which URL was echoed. +func RedactCredentialsInText(text string) string { + return credentialInURL.ReplaceAllString(text, "${1}") +} + // RedactURLOrPath renders a remote for display with any credentials removed, // accepting values that are not URLs at all. // diff --git a/cmd/entire/cli/plugin_gitremote.go b/cmd/entire/cli/plugin_gitremote.go index 9b6e07821e..646230d725 100644 --- a/cmd/entire/cli/plugin_gitremote.go +++ b/cmd/entire/cli/plugin_gitremote.go @@ -173,10 +173,8 @@ func redactURL(raw string) string { // redactCredentials removes userinfo from any URL appearing in free text, for // git's stderr where we do not know which URL was echoed. -var credentialInURL = regexp.MustCompile(`([a-zA-Z][a-zA-Z0-9+.-]*://)[^/@\s]*@`) - func redactCredentials(text string) string { - return credentialInURL.ReplaceAllString(text, "${1}") + return gitremote.RedactCredentialsInText(text) } // gitURLSchemes are the transports a plugin or index repository URL may use in diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index aa3291f3cc..200376831c 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -421,7 +421,7 @@ func (s *ManualCommitStrategy) prePushCheckpointRefs(ctx context.Context, ps pus return nil } - if flushed, err := flushCheckpointRefsQueue(ctx, repo, ps); err == nil { + if flushed, err := flushCheckpointRefsQueue(ctx, repo, ps, true); err == nil { // Delivered, and only if something actually was: an empty queue pushed // nothing, so it must not move the election or announce that it had. if pendingCapture != "" && flushed > 0 { @@ -462,7 +462,7 @@ func PushQueuedCheckpointRefs(ctx context.Context, repo *git.Repository, remote // wrong problem. The wrapped error says which it was. return 0, false, fmt.Errorf("checkpoint refs stay queued: %w", opfErr) } - pushed, err = flushCheckpointRefsQueue(ctx, repo, ps) + pushed, err = flushCheckpointRefsQueue(ctx, repo, ps, false) return pushed, false, err } @@ -472,9 +472,10 @@ func PushQueuedCheckpointRefs(ctx context.Context, repo *git.Repository, remote // // Shared by the git-refs pre-push path (which logs and ignores the error to // never block the user's push) and the migration command's opt-in push (which -// surfaces it). Stale entries — refs no longer present locally — are pruned so +// surfaces it). boundBatch puts the batch push under checkpointFlushBudget too; +// the pre-push path sets it, the explicit migration push does not. Stale entries — refs no longer present locally — are pruned so // they don't block the queue forever. -func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps pushSettings) (int, error) { +func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps pushSettings, boundBatch bool) (int, error) { queue, err := checkpoint.PushQueueForRepo(ctx, repo) if err != nil { return 0, fmt.Errorf("resolve push queue: %w", err) @@ -514,116 +515,112 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push fmt.Fprintf(os.Stderr, "[entire] Pushing %d checkpoint ref(s) to %s...", len(existing), dest.display()) stop := startProgressDots(os.Stderr) - // Fast path: push all refs in one round-trip (fast-forward-only). If every - // ref was up to date or fast-forwarded, we're done. - batchErr := batchPushRefs(pushCtx, dest.target, existing) - if batchErr == nil { - stop(" done") - if removeErr := queue.Remove(existing); removeErr != nil { + // One budget for the whole flush, opened before the batch: a backlog upload + // is as able to hang the user's push as a walk of per-ref retries. Bounded; + // see checkpointFlushBudget and maxConsecutiveRefPushFailures. + flushCtx, cancelFlush := context.WithTimeout(pushCtx, checkpointFlushBudget) + defer cancelFlush() + batchCtx := flushCtx + if !boundBatch { + batchCtx = pushCtx + } + + // Fast path: push the refs a chunk per round-trip (fast-forward-only). + batch := pushRefChunks(batchCtx, dest.target, existing, func(landed []plumbing.ReferenceName) { + if removeErr := queue.Remove(landed); removeErr != nil { logging.Warn(ctx, "git-refs push: clear pushed refs from queue failed", slog.String("error", removeErr.Error())) } - return len(existing), nil + }) + if len(batch.failed) == 0 && len(batch.untried) == 0 { + stop(" done") + return batch.landed, nil + } + if len(batch.failed) == 0 { + // Every attempted chunk landed, but the budget ran out first. Nothing + // failed, so this is progress rather than an error. + stop(fmt.Sprintf(" pushed %d of %d", batch.landed, len(existing))) + fmt.Fprintf(os.Stderr, "[entire] Stopped pushing: %s; %d checkpoint ref(s) stay queued for the next push.\n", + batch.stopReason, len(batch.untried)) + logging.Warn(ctx, "git-refs push: batch push stopped early; remaining refs stay queued", + slog.String("reason", batch.stopReason), slog.Int("pushed", batch.landed), + slog.Int("queued", len(existing))) + return batch.landed, nil + } + if batch.unreachable != "" { + stop(" failed") + } else { + stop("") } - stop("") // Non-interactive SSH auth failures cannot be fixed by per-ref // fetch+replay. Surface the same actionable hint as the v1 doPushRef path // (issue #1523) instead of only logging to .entire/logs/. - // Deliberately does not print batchErr: it carries git's own output, and - // this runs inside the user's `git push`. The hint below is the actionable - // part; the full error reaches .entire/logs via the caller, which logs the - // error this branch returns. - if nonInteractiveSSHAuthFailure(pushCtx, batchErr) { + // Deliberately does not print the batch error: it carries git's own output, + // and this runs inside the user's `git push`. The hint below is the + // actionable part; the full error reaches .entire/logs via the caller, which + // logs the error this branch returns. + if batch.sshAuthFailed { fmt.Fprintln(os.Stderr, "[entire] Warning: couldn't push checkpoint refs (SSH authentication failed).") printNonInteractiveSSHAuthHint() if dest.checkpointRemote { printCheckpointRemoteHint(dest.target) } - return 0, batchErr + return batch.landed, batch.firstErr + } + + // Nothing landed and git could not even connect: the per-ref fallback would + // fail the same way, one connect timeout per ref. Unlike the branches above, + // name the cause: it is one line from ssh or curl (credentials redacted, see + // remote.UnreachableRemoteLine), and without it the user sees a slow push + // fail with no reason. No rotation: no ref failed on its own account. + if batch.unreachable != "" { + fmt.Fprintf(os.Stderr, "[entire] Couldn't reach %s: %s\n", dest.display(), batch.unreachable) + fmt.Fprintf(os.Stderr, "[entire] %d checkpoint ref(s) stay queued for the next push.\n", len(existing)) + if dest.checkpointRemote { + printCheckpointRemoteHint(dest.target) + } + return batch.landed, batch.firstErr } - // At least one ref was rejected — typically a non-fast-forward divergence - // (the same checkpoint re-written on another machine). Retry per ref with - // fetch+replay recovery, and remove from the queue only the refs that land - // (a genuine cherry-pick conflict leaves that ref queued for a later push, - // never force-overwriting the remote). + // At least one chunk failed — typically a non-fast-forward divergence (the + // same checkpoint re-written on another machine). Retry its refs one at a + // time with fetch+replay recovery, and remove from the queue only the refs + // that land (a genuine cherry-pick conflict leaves that ref queued for a + // later push, never force-overwriting the remote). Refs of the chunk that did + // land are cheap here: their push reports them up to date. // Deliberately names no cause: the batch fails on divergence, but just as // often on an unreachable or unauthorized destination. Telling a user with a // dead remote that their refs "diverged" — or were "rejected", which equally // implies the remote answered — sends them after the wrong problem. - // The only account of a wholesale failure. The per-ref retries below - // re-derive a *rejection* reason, but a transport failure — an unreachable - // remote, a stalled SSH connection — matches nothing in - // checkpointRefRejectionReason, and this path returns firstErr rather than - // batchErr, so without this line the cause of a 300-ref failure reached - // neither the terminal nor .entire/logs. Logged, not printed: the one - // actionable line printed below is the useful part. - logging.Warn(ctx, "git-refs push: batch checkpoint ref push failed; retrying individually", - slog.Int("refs", len(existing)), slog.String("error", batchErr.Error())) - - fmt.Fprintf(os.Stderr, "[entire] Checkpoint ref push failed; retrying %d ref(s) individually...", len(existing)) + fmt.Fprintf(os.Stderr, "[entire] Checkpoint ref push failed; retrying %d ref(s) individually...", len(batch.failed)) stop = startProgressDots(os.Stderr) - pushed := make([]plumbing.ReferenceName, 0, len(existing)) - var firstErr error - var rejectionWarning string - // Bounded; see checkpointFlushBudget and maxConsecutiveRefPushFailures. - flushCtx, cancelFlush := context.WithTimeout(pushCtx, checkpointFlushBudget) - defer cancelFlush() - consecutiveFailures := 0 - attempted := 0 - var abortReason string - var failed []plumbing.ReferenceName - for _, ref := range existing { - // Before the attempt but never before the first: a flush always tries at - // least one ref, and only aborts while refs are left to skip. - if attempted > 0 { - if abortReason = flushAbortReason(flushCtx, consecutiveFailures); abortReason != "" { - logging.Warn(ctx, "git-refs push: individual retry stopped early; remaining refs stay queued", - slog.String("reason", abortReason), slog.Int("attempted", attempted), - slog.Int("queued", len(existing))) - break - } - } - attempted++ - if err := pushCheckpointRefWithRecovery(flushCtx, dest.target, ref); err != nil { - consecutiveFailures++ - logging.Warn(ctx, "git-refs push: checkpoint ref push/sync failed; left queued, not overwritten", - slog.String("ref", ref.String()), slog.String("error", err.Error())) - if nonInteractiveSSHAuthFailure(flushCtx, err) { - printNonInteractiveSSHAuthHint() - } - if rejectionWarning == "" { - if reason := checkpointRefRejectionReason(err); reason != "" { - rejectionWarning = fmt.Sprintf("[entire] Warning: checkpoint ref %s remains queued (showing one rejection):\n%s", ref, reason) - } - } - if firstErr == nil { - firstErr = err - } - failed = append(failed, ref) - } else { - consecutiveFailures = 0 - pushed = append(pushed, ref) - } + retry := retryRefsIndividually(ctx, flushCtx, dest.target, batch.failed, len(existing)) + pushed, failed, firstErr, abortReason := retry.pushed, retry.failed, retry.firstErr, retry.abortReason + attempted := len(pushed) + len(failed) + totalPushed := batch.landed + len(pushed) + stop(fmt.Sprintf(" pushed %d of %d", totalPushed, len(existing))) + // The fallback reached every failed ref, but the batch stopped before + // trying the rest of the queue: report that stop instead. + if abortReason == "" && len(batch.untried) > 0 { + abortReason = batch.stopReason } - stop(fmt.Sprintf(" pushed %d of %d", len(pushed), len(existing))) // Printed before the rejection warning so the more specific reason lands // closest to the prompt. if abortReason != "" { // Everything this flush did not land stays queued, not just the refs it - // never reached: only `pushed` is removed, so the ones that were + // never reached: only landed refs are removed, so the ones that were // attempted and failed are still there too. fmt.Fprintf(os.Stderr, "[entire] Stopped retrying: %s; %d checkpoint ref(s) stay queued for the next push.\n", - abortReason, len(existing)-len(pushed)) + abortReason, len(existing)-totalPushed) } // One actionable example per flush, after the progress line. Label it as // such: other queued refs may have different causes (all are logged). // Preserve Git's line breaks rather than printing the single-line log error. // Do not print the batch error too, or diagnose speculative recovery as // divergence. - if rejectionWarning != "" { - fmt.Fprintln(os.Stderr, rejectionWarning) + if retry.rejectionWarning != "" { + fmt.Fprintln(os.Stderr, retry.rejectionWarning) } if err := queue.Remove(pushed); err != nil { logging.Warn(ctx, "git-refs push: clear pushed refs from queue failed", @@ -656,8 +653,58 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push // Counts attempts, not the queue: refs skipped by an early abort were // never tried, and reporting them as failures would overstate what the // remote actually refused. - return len(pushed), fmt.Errorf("%d of %d attempted checkpoint refs failed to push: %w", + return totalPushed, fmt.Errorf("%d of %d attempted checkpoint refs failed to push: %w", attempted-len(pushed), attempted, firstErr) } - return len(pushed), nil + return totalPushed, nil +} + +// refRetryResult is what the per-ref fallback of a flush left behind. +type refRetryResult struct { + pushed []plumbing.ReferenceName + failed []plumbing.ReferenceName // attempted and failed; unreached refs are in neither + firstErr error + abortReason string + rejectionWarning string +} + +// retryRefsIndividually pushes refs one at a time with fetch+replay recovery +// under flushCtx, stopping early per flushAbortReason. ctx is used for logging. +func retryRefsIndividually(ctx, flushCtx context.Context, target string, refs []plumbing.ReferenceName, queued int) refRetryResult { + var res refRetryResult + consecutiveFailures := 0 + for i, ref := range refs { + // Before the attempt but never before the first: a flush always tries at + // least one ref, and only aborts while refs are left to skip. + if i > 0 { + if res.abortReason = flushAbortReason(flushCtx, consecutiveFailures); res.abortReason != "" { + logging.Warn(ctx, "git-refs push: individual retry stopped early; remaining refs stay queued", + slog.String("reason", res.abortReason), slog.Int("attempted", i), + slog.Int("queued", queued)) + break + } + } + err := pushCheckpointRefWithRecovery(flushCtx, target, ref) + if err == nil { + consecutiveFailures = 0 + res.pushed = append(res.pushed, ref) + continue + } + consecutiveFailures++ + logging.Warn(ctx, "git-refs push: checkpoint ref push/sync failed; left queued, not overwritten", + slog.String("ref", ref.String()), slog.String("error", err.Error())) + if nonInteractiveSSHAuthFailure(flushCtx, err) { + printNonInteractiveSSHAuthHint() + } + if res.rejectionWarning == "" { + if reason := checkpointRefRejectionReason(err); reason != "" { + res.rejectionWarning = fmt.Sprintf("[entire] Warning: checkpoint ref %s remains queued (showing one rejection):\n%s", ref, reason) + } + } + if res.firstErr == nil { + res.firstErr = err + } + res.failed = append(res.failed, ref) + } + return res } diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 79b832fb0a..58bc50754e 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -45,7 +45,7 @@ func partitionLocalRefs(repo *git.Repository, refs []plumbing.ReferenceName) (ex } // batchPushRefs pushes all of refs to target in a single git push, -// fast-forward-only (NOT a force push). Batching keeps a backfill of many refs to +// fast-forward-only (NOT a force push). Batching keeps a chunk of many refs to // one network round-trip. Per-checkpoint refs normally advance by fast-forward // (each write parents on the prior tip), so the common case succeeds; a // non-fast-forward update — genuine divergence, e.g. the same checkpoint written @@ -69,6 +69,90 @@ func batchPushRefs(ctx context.Context, target string, refs []plumbing.Reference return nil } +// chunkPushResult is what the chunked batch phase of a flush left behind. +type chunkPushResult struct { + landed int // refs in chunks that pushed cleanly + failed []plumbing.ReferenceName // refs of chunks that failed, for the per-ref fallback + untried []plumbing.ReferenceName // refs never attempted because the phase stopped + stopReason string // why untried is non-empty + firstErr error + sshAuthFailed bool + // unreachable is the line of git output showing the remote could not be + // reached at all; set only when no chunk had landed. See + // remote.UnreachableRemoteLine. + unreachable string +} + +// pushRefChunks batch-pushes refs a chunk at a time (see +// checkpointRefPushChunkSize), calling onLanded with each chunk that lands so +// the caller can dequeue it before a later chunk is cut. A failed chunk does not +// stop the phase — a rejection fails only its own chunk — but an SSH auth +// failure, an expired or cancelled ctx, or maxConsecutiveChunkPushFailures in a +// row does, and so does an unreachable remote before any chunk landed: per-ref +// retries cannot reach it either, and each would wait out its own connect +// timeout. The first chunk is always attempted, matching the per-ref fallback. +func pushRefChunks(ctx context.Context, target string, refs []plumbing.ReferenceName, onLanded func([]plumbing.ReferenceName)) chunkPushResult { + var res chunkPushResult + consecutive := 0 + for start := 0; start < len(refs); start += checkpointRefPushChunkSize { + if start > 0 { + if reason := chunkStopReason(ctx, consecutive); reason != "" { + res.untried = refs[start:] + res.stopReason = reason + logging.Warn(ctx, "git-refs push: batch push stopped early", + slog.String("reason", reason), slog.Int("untried", len(res.untried))) + return res + } + } + chunk := refs[start:min(start+checkpointRefPushChunkSize, len(refs))] + err := batchPushRefs(ctx, target, chunk) + if err == nil { + consecutive = 0 + res.landed += len(chunk) + onLanded(chunk) + continue + } + consecutive++ + // The only account of a wholesale failure. The per-ref retries re-derive + // a *rejection* reason, but a transport failure — an unreachable remote, + // a stalled SSH connection — matches nothing in + // checkpointRefRejectionReason, so without this line its cause reached + // neither the terminal nor .entire/logs. Logged, not printed: the + // caller's one actionable line is the useful part. + logging.Warn(ctx, "git-refs push: batch checkpoint ref push failed; retrying individually", + slog.Int("refs", len(chunk)), slog.String("error", err.Error())) + if res.firstErr == nil { + res.firstErr = err + } + res.failed = append(res.failed, chunk...) + if nonInteractiveSSHAuthFailure(ctx, err) { + res.sshAuthFailed = true + res.untried = refs[start+len(chunk):] + return res + } + // A remote that already took a chunk this flush was reachable; a later + // connect failure is transient, and the per-ref fallback may still land. + if line, ok := remote.UnreachableRemoteLine(err); ok && res.landed == 0 { + res.unreachable = line + res.untried = refs[start+len(chunk):] + return res + } + } + return res +} + +// chunkStopReason reports why the batch phase should stop before its next +// chunk, or "" to continue. Same bare phrasing as flushAbortReason. +func chunkStopReason(ctx context.Context, consecutiveFailures int) string { + if reason := flushAbortReason(ctx, 0); reason != "" { + return reason + } + if consecutiveFailures >= maxConsecutiveChunkPushFailures { + return fmt.Sprintf("%d consecutive failed batches", consecutiveFailures) + } + return "" +} + // pushCheckpointRefWithRecovery pushes a single checkpoint ref fast-forward-only; // confirmed remote policy/hook rejections return immediately. Other failures — // typically the ref diverged on the remote (the same checkpoint re-written @@ -201,10 +285,12 @@ func displayPushTarget(target string) string { // can shrink it. var checkpointPushBudget = 2 * time.Minute -// checkpointFlushBudget bounds the individual-retry fallback in -// flushCheckpointRefsQueue as a whole, which checkpointPushBudget cannot: that -// one caps a single ref, and the fallback walks the queue serially, so a backlog -// multiplies it. A healthy push costs roughly one SSH round-trip per ref, so a +// checkpointFlushBudget bounds a pre-push flushCheckpointRefsQueue as a whole — +// the chunked batch push and the individual-retry fallback share it — which +// checkpointPushBudget cannot: that one caps a single ref, and the fallback walks +// the queue serially, so a backlog multiplies it. The batch needs the bound too: +// it carries the whole backlog, so a slow uplink or a stalled connection would +// otherwise hold the user's git push for as long as the upload takes. A healthy push costs roughly one SSH round-trip per ref, so a // few hundred queued refs is already tens of minutes of a `git push` that looks // hung; when the destination is unreachable every ref instead pays the full // per-ref budget and the same queue runs for hours. Refs that do not fit stay @@ -215,9 +301,27 @@ var checkpointPushBudget = 2 * time.Minute // context.WithTimeout keeps the earlier of parent and child deadlines, so the // per-ref checkpointPushBudget automatically shrinks to whatever is left. // +// Shared, so a batch that spends the budget leaves the fallback none: its one +// guaranteed attempt then fails at once, and the refs stay queued for the next +// push. The explicit migration push (PushQueuedCheckpointRefs) leaves the batch +// unbounded; the user asked for that upload and is waiting on it. +// // Declared as a var so tests can shrink it. var checkpointFlushBudget = 2 * time.Minute +// checkpointRefPushChunkSize is how many queued refs one batch push carries. +// Chunks are what make the flush budget bound progress rather than discard it: +// each chunk that lands leaves the queue at once, so a backlog too large for one +// budget drains over several pushes instead of being cut at the same point by +// every one of them. Declared as a var so tests can shrink it. +var checkpointRefPushChunkSize = 25 + +// maxConsecutiveChunkPushFailures stops the batch phase once this many chunks in +// a row have failed. A rejection fails only its own chunk, so batching carries on +// past one; several in a row point at the destination, and the per-ref fallback +// already probes that with its own cap. +const maxConsecutiveChunkPushFailures = 2 + // maxConsecutiveRefPushFailures stops the individual-retry fallback once this // many refs in a row have failed. The fallback exists to isolate the odd // diverged or blocked ref from an otherwise pushable queue; once several fail in diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index 0585f2b37b..f122f2db64 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -270,3 +270,219 @@ func TestFlushCheckpointRefs_BudgetAbortRotatesToo(t *testing.T) { assert.Equal(t, append(append([]plumbing.ReferenceName{}, refs[1:]...), refs[0]), remaining, "the attempted ref moves to the back so the next push starts somewhere new") } + +func shrinkRefPushChunkSize(t *testing.T, n int) { + t.Helper() + restore := checkpointRefPushChunkSize + checkpointRefPushChunkSize = n + t.Cleanup(func() { checkpointRefPushChunkSize = restore }) +} + +// installCountingHook accepts every push, recording one line per push in +// countFile, and sleeps on every push after the first sleepAfter. +func installCountingHook(t *testing.T, bareDir, countFile string, sleepAfter int) { + t.Helper() + hook := "#!/bin/sh\ncat >/dev/null\n" + + "n=$(cat '" + countFile + "' 2>/dev/null | wc -l)\necho attempt >> '" + countFile + "'\n" + if sleepAfter > 0 { + hook += fmt.Sprintf("[ \"$n\" -ge %d ] && sleep 30\n", sleepAfter) + } + hook += "exit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) +} + +// installBlockRefHook declines any push whose ref updates include blockedRef — +// the whole push, as a ruleset does, so the blocked ref's chunk fails with it. +func installBlockRefHook(t *testing.T, bareDir, blockedRef string) { + t.Helper() + hook := "#!/bin/sh\nblocked=0\nwhile read -r old new ref; do\n" + + " [ \"$ref\" = '" + blockedRef + "' ] && blocked=1\ndone\n" + + "if [ \"$blocked\" = 1 ]; then echo '" + checkpointRejectReason + "' >&2; exit 1; fi\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) +} + +// TestFlushCheckpointRefs_PushesBacklogInChunks pins the chunked batch: a +// backlog larger than one chunk goes out a chunk per push and fully drains. +func TestFlushCheckpointRefs_PushesBacklogInChunks(t *testing.T) { + shrinkRefPushChunkSize(t, 3) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 8) + countFile := filepath.Join(t.TempDir(), "attempts") + prepareGitRefsPrePush(t, workDir, bareDir) + installCountingHook(t, bareDir, countFile, 0) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + assert.Equal(t, 3, countedAttempts(t, countFile), "8 refs in chunks of 3 is three pushes") + assert.Contains(t, output, "Pushing 8 checkpoint ref(s)") + assert.Contains(t, output, " done") + for _, ref := range refs { + assert.Equal(t, refHashOf(t, repo, ref), remoteRefHash(t, bareDir, ref)) + } + remaining, err := queue.Drain() + require.NoError(t, err) + assert.Empty(t, remaining) +} + +// TestFlushCheckpointRefs_BudgetKeepsLandedChunks is the hang this bound +// exists for: a batch upload that stalls must not hold the user's git push past +// the flush budget, and the chunks that landed before it must leave the queue so +// the next push starts after them instead of re-sending the whole backlog. +func TestFlushCheckpointRefs_BudgetKeepsLandedChunks(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 6) + countFile := filepath.Join(t.TempDir(), "attempts") + prepareGitRefsPrePush(t, workDir, bareDir) + installCountingHook(t, bareDir, countFile, 1) // first push lands, the rest stall + + restoreBudget := checkpointFlushBudget + checkpointFlushBudget = 3 * time.Second + t.Cleanup(func() { checkpointFlushBudget = restoreBudget }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + start := time.Now() + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + elapsed := time.Since(start) + output := restore() + + assert.Less(t, elapsed, 25*time.Second, "a stalled upload must be cut by the budget, not waited out") + assert.Contains(t, output, "exhausted") + assert.Contains(t, output, "4 checkpoint ref(s) stay queued") + for _, ref := range refs[:2] { + assert.Equal(t, refHashOf(t, repo, ref), remoteRefHash(t, bareDir, ref), "the first chunk landed") + } + remaining, err := queue.Drain() + require.NoError(t, err) + assert.ElementsMatch(t, refs[2:], remaining, "landed chunks leave the queue; the rest stay") +} + +// TestFlushCheckpointRefs_RejectedChunkDoesNotStopLaterChunks pins that a +// rejection fails only its own chunk: later chunks still go out in the batch, +// and the rejected chunk's healthy refs land through the per-ref fallback. +func TestFlushCheckpointRefs_RejectedChunkDoesNotStopLaterChunks(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 6) + prepareGitRefsPrePush(t, workDir, bareDir) + installBlockRefHook(t, bareDir, refs[0].String()) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + assert.Contains(t, output, "retrying 2 ref(s) individually", "only the rejected chunk falls back") + assert.Contains(t, output, "pushed 5 of 6") + for _, ref := range refs[1:] { + assert.Equal(t, refHashOf(t, repo, ref), remoteRefHash(t, bareDir, ref)) + } + assertRefsAbsentFromRemote(t, bareDir, refs[:1], "the blocked ref must stay local") + remaining, err := queue.Drain() + require.NoError(t, err) + assert.Equal(t, refs[:1], remaining) +} + +// TestFlushCheckpointRefs_StopsBatchAfterConsecutiveChunkFailures pins the +// batch phase's own cap: a remote refusing every push refuses every chunk, so +// the batch stops instead of sending the whole backlog to be refused. +func TestFlushCheckpointRefs_StopsBatchAfterConsecutiveChunkFailures(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 8) + countFile := filepath.Join(t.TempDir(), "attempts") + prepareGitRefsPrePush(t, workDir, bareDir) + installCheckpointRejectHook(t, bareDir, false, countFile) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + // Two chunk pushes, then one per-ref retry for each of their four refs. + assert.Equal(t, maxConsecutiveChunkPushFailures+4, countedAttempts(t, countFile)) + assert.Contains(t, output, fmt.Sprintf("%d consecutive failed batches", maxConsecutiveChunkPushFailures)) + assert.Contains(t, output, "8 checkpoint ref(s) stay queued") + remaining, err := queue.Drain() + require.NoError(t, err) + assert.ElementsMatch(t, refs, remaining) +} + +// TestPushQueuedCheckpointRefs_BatchNotBudgeted pins the migration push's +// exemption: the user asked for that upload and waits on it, so the batch is +// not cut by the pre-push budget. +func TestPushQueuedCheckpointRefs_BatchNotBudgeted(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 5) + prepareGitRefsPrePush(t, workDir, bareDir) + + restoreBudget := checkpointFlushBudget + checkpointFlushBudget = time.Nanosecond + t.Cleanup(func() { checkpointFlushBudget = restoreBudget }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + pushed, err := flushCheckpointRefsQueue(t.Context(), repo, pushSettings{remote: bareDir}, false) + restore() + require.NoError(t, err) + assert.Equal(t, len(refs), pushed) + remaining, err := queue.Drain() + require.NoError(t, err) + assert.Empty(t, remaining) +} + +// TestFlushCheckpointRefs_UnreachableRemoteStopsAtOnce pins the dead-host +// bound: when git cannot even connect, neither later chunks nor per-ref retries +// can either, so the flush makes one attempt, says why, and keeps the queue. +func TestFlushCheckpointRefs_UnreachableRemoteStopsAtOnce(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, _, refs := setupRepoWithNCheckpointRefs(t, 6) + prepareGitRefsPrePush(t, workDir, "ssh://git@example.invalid/repo.git") + + // A fake ssh client: counts connections and fails the way OpenSSH does + // against a refused port, without touching the network. Named "ssh" so git + // takes it for OpenSSH and skips its "-G" probe of unrecognized clients. + countFile := filepath.Join(t.TempDir(), "attempts") + fakeSSH := filepath.Join(t.TempDir(), "ssh") + script := "#!/bin/sh\necho attempt >> '" + countFile + "'\n" + + "echo 'ssh: connect to host example.invalid port 22: Connection refused' >&2\nexit 255\n" + require.NoError(t, os.WriteFile(fakeSSH, []byte(script), 0o755)) + t.Setenv("GIT_SSH_COMMAND", fakeSSH) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + assert.Equal(t, 1, countedAttempts(t, countFile), "no further chunk or per-ref retry against a dead host") + assert.Contains(t, output, "Couldn't reach origin: ssh: connect to host example.invalid port 22: Connection refused") + assert.Contains(t, output, "6 checkpoint ref(s) stay queued") + assert.NotContains(t, output, "retrying") + remaining, err := queue.Drain() + require.NoError(t, err) + assert.Equal(t, refs, remaining, "nothing is dropped or reordered") +} diff --git a/cmd/entire/cli/strategy/refs_push_test.go b/cmd/entire/cli/strategy/refs_push_test.go index dc03bb7806..1cd4703f3d 100644 --- a/cmd/entire/cli/strategy/refs_push_test.go +++ b/cmd/entire/cli/strategy/refs_push_test.go @@ -166,7 +166,7 @@ func TestPushCheckpointRefWithRecovery_MergesDivergedRef(t *testing.T) { // pushes are rejected, then recovery replays C3's delta onto C2. queue := enqueueRefs(t, repo, []plumbing.ReferenceName{ref}) restore := captureStderr(t) - pushed, pushErr := flushCheckpointRefsQueue(ctx, repo, pushSettings{remote: bareDir}) + pushed, pushErr := flushCheckpointRefsQueue(ctx, repo, pushSettings{remote: bareDir}, true) output := restore() require.NoError(t, pushErr, "diverged ref should be recovered by fetch+replay, not rejected") assert.Equal(t, 1, pushed) diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index b569fb081b..5074796837 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -101,9 +101,8 @@ Rewrites are atomic (temp file + rename under the lock) so a concurrent reader n 1. **Drains** the queue. 2. **Partitions** the drained refs into those that still exist locally and stale ones (dropped from the queue). -3. **Batch-pushes** the existing refs in one network round-trip (`batchPushRefs`, `strategy/push_common.go`). -4. On success, **removes** the pushed refs from the queue. -5. On a batch failure (typically a non-fast-forward rejection), **falls back to per-ref recovery** (`pushCheckpointRefWithRecovery`) and removes from the queue only the refs that land. +3. **Batch-pushes** the existing refs a chunk at a time, one network round-trip per `checkpointRefPushChunkSize` refs (`pushRefChunks` → `batchPushRefs`, `strategy/push_common.go`), **removing** each chunk from the queue as soon as it lands. +4. On a chunk failure (typically a non-fast-forward rejection), carries on with the next chunk, then **falls back to per-ref recovery** (`pushCheckpointRefWithRecovery`) for the failed chunks' refs only, removing from the queue only the refs that land. Confirmed remote policy/hook rejections skip fetch/replay: replay cannot fix a policy block and would needlessly rewrite local commits if the ref already exists @@ -119,11 +118,15 @@ without additional secret redaction of the remote's diagnostic output. Plain non SSH authentication failures retain their dedicated hint. Failures remain queued and **never fail the user's git push**. +The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch that spends the budget leaves the fallback none, so its one guaranteed attempt fails at once and the refs stay queued. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it. Under the pre-push hook, OpenSSH also gets `ConnectTimeout` (`withBatchModeSSH`), so an unreachable host fails within seconds instead of spending the budget on a kernel connect timeout. + +A remote that cannot be reached at all ends the flush after one attempt. When nothing has landed and a chunk's git output carries a connect-phase failure from ssh or curl (`remote.UnreachableRemoteLine`: unresolvable host, failed connect), later chunks and the per-ref fallback would each fail the same way after their own connect timeout, so the flush stops, prints that one line — credentials redacted, control bytes stripped, capped — and leaves the whole queue in order. The match is deliberately narrow: no bare strerror text (curl's is localized), no mid-transfer timeouts (the host was reachable), and no `remote:` lines (a reachable remote's own hook failing to resolve a host). A miss only costs the old path, the bounded fallback. + The fallback is bounded. It runs under a `checkpointFlushBudget` context deadline — so an expired budget cuts a ref that is already hung rather than waiting out its own `checkpointPushBudget` first — and additionally stops after `maxConsecutiveRefPushFailures` refs fail in a row. Either way it never aborts before at least one ref has been attempted. A remote that is refusing or unreachable fails every ref the same way, and each ref in the fallback costs at least one network round-trip, so walking a large queue to the end turns one failed push into an apparently hung one for minutes or hours. Skipped refs stay queued and go out on the next push, and a success resets the consecutive count so one blocked checkpoint does not strand the refs queued behind it. The abort prints one line naming what stopped the retry and how many refs remain. An early abort also rotates that flush's failures to the back of the queue (`PushQueue.Rotate`). The queue drains in first-seen order, so without it a prefix that always fails — five checkpoints blocked by a ruleset, say — is retried in the same order on every push and the healthy refs behind it are never attempted at all. Rotation is skipped after an interruption, where the failures say nothing about the refs. -Two diagnostics make a wholesale failure readable. The batch error is logged before the fallback runs: the per-ref retries re-derive a *rejection* reason, but a transport failure — an unreachable remote, a stalled connection — matches none of them, so without that line the cause reached neither the terminal nor `.entire/logs`. And `remote.Fetch` now annotates its error with git's own output — redacted and capped by the same `errWithGitOutput` helper the push path uses — instead of the strategy layer substituting that output for the error, which reported a contentless `fetch failed:` per ref whenever git was killed before writing anything. +Two diagnostics make a wholesale failure readable. Each failed chunk's error is logged before the fallback runs: the per-ref retries re-derive a *rejection* reason, but a transport failure — an unreachable remote, a stalled connection — matches none of them, so without that line the cause reached neither the terminal nor `.entire/logs`. And `remote.Fetch` now annotates its error with git's own output — redacted and capped by the same `errWithGitOutput` helper the push path uses — instead of the strategy layer substituting that output for the error, which reported a contentless `fetch failed:` per ref whenever git was killed before writing anything. ### Non-force, fast-forward-only diff --git a/docs/development/checkpoint-implementation.md b/docs/development/checkpoint-implementation.md index a30cd4c5a0..7f301be26c 100644 --- a/docs/development/checkpoint-implementation.md +++ b/docs/development/checkpoint-implementation.md @@ -67,7 +67,7 @@ The manual-commit strategy (`manual_commit*.go`) does not modify the active bran - **Orphaned session state cleanup** (`isOrphanedSessionState`) - `listAllSessionStates` deletes a state only when it is ended (`State.IsEnded`: ENDED phase or `EndedAt` stamped), never condensed (no `LastCheckpointID`), and has no pending work (`!HasPendingWork()`). Legacy empty-phase records normalize to IDLE when listed, so they are no longer deleted here (they previously were) and instead remain until `StaleSessionThreshold` expiry. An IDLE never-condensed state is either a live session between turns — a read-only turn, or the turn right after a linked commit — or belongs to the exited-owner finalizer when `OwnerExited` is true. The store is listed by every worktree's hooks, so deleting either shape re-initialised live sessions from zero or erased the ownership evidence before `finalizeExitedSessions` could end the session cleanly. IDLE states therefore remain for lifecycle finalization or `StaleSessionThreshold` expiry. - PrePush hook can push `entire/checkpoints/v1` branch alongside user pushes - **Git-refs push failures stay fail-soft**: confirmed remote policy/hook rejections skip fetch/replay so repeated blocked pushes do not rewrite local commits. Other failures still attempt recovery; failed fetch/replay preserves the original push cause, not a presumed divergence. Confirmed remote rejections print one bounded example per flush with Git's line breaks intact (log errors remain single-line), without additional secret redaction of the remote's diagnostic output; unsuccessful refs stay queued. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). -- **The git-refs per-ref fallback is bounded**: it runs under a `checkpointFlushBudget` context deadline and stops after `maxConsecutiveRefPushFailures` consecutive failures — always after at least one attempt, with skipped refs left queued for the next push. The batch error is logged before the fallback runs so a transport failure, which `checkpointRefRejectionReason` deliberately does not match, still names its cause; and `remote.Fetch` annotates its error with git's redacted output via `errWithGitOutput` — deliberately not a `PushError`, which is how a refused push is recognised — rather than the caller substituting that output for the error. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). +- **The git-refs pre-push flush is bounded end to end**: the batch pushes `checkpointRefPushChunkSize` refs per round-trip, dequeuing each chunk as it lands, and in the pre-push path shares one `checkpointFlushBudget` deadline with the per-ref fallback (the explicit migration push leaves the batch unbounded). The batch stops after `maxConsecutiveChunkPushFailures` failed chunks in a row, or after the first chunk that fails to connect at all (`remote.UnreachableRemoteLine`; skips the fallback and prints that one redacted line), and the fallback after `maxConsecutiveRefPushFailures` failed refs — each always after at least one attempt, with skipped refs left queued for the next push. Each failed chunk's error is logged before the fallback runs so a transport failure, which `checkpointRefRejectionReason` deliberately does not match, still names its cause; and `remote.Fetch` annotates its error with git's redacted output via `errWithGitOutput` — deliberately not a `PushError`, which is how a refused push is recognised — rather than the caller substituting that output for the error. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). - **Single checkpoint sync remote** - checkpoint data syncs only to the elected checkpoint sync remote (`strategy_options.checkpoint_push_remote` setting → captured election → `origin` → sole remote → first remote in `.git/config` order; fail-closed only when the setting names a missing remote — the captured tier is fail-soft), on both the git-branch and git-refs backends. **Capture**: during pre-push, a push whose target agrees with the branch's declared push destination (`branch..pushRemote` → `remote.pushDefault` → `branch..remote`) and differs from the current election persists that remote as the captured election (git common dir, `entire-checkpoint-sync-remotes.json`), announces it on stderr, and the same push carries the checkpoints — see `strategy/checkpoint_sync_capture.go`. Declaration alone is deliberately **not** a tier: electing from tracking config at rest silently drops checkpoint sync on every push to a different remote (`git push HEAD`, a `git clone -o base` pushing checkpoints to a separately added `origin`, any repo with `remote.pushDefault` — the `74e239a9` regression), and a bare push to an undeclared remote never elects (the pre-single-remote transcript leak). Phase-1 capture rules: at most one captured remote, first still-configured capture sticks (no per-push ping-pong in mixed-habit repos; a captured remote that was renamed or removed no longer vetoes the next capture), explicit `checkpoint_push_remote` outranks and disables capture. Pushes to any other remote or to a raw URL never carry checkpoint data; the dedicated `checkpoint_remote` URL mode is exempt. `entire status` shows the sync destination, its source (`observed` renders as "follows your branch's push destination"), and an unpushed-checkpoint count. - **A refused `checkpoint_remote` says so where it will be seen** - `checkpointRemoteIsInherited` refuses a committed `checkpoint_remote` whose owner does not match every remote identifying this repo, and used to say so only in a WARN. The visible symptom — checkpoints arriving in the code repository — is a working setup, just not the one that was asked for, so the rejection now reaches the user in `entire status` (text and `--json`, with `checkpoint_remote_ignored_verdict`) and `entire enable`, which take one verdict from `ignoredCheckpointRemote` and print the same sentence — "Checkpoints sync to X, not to the configured checkpoint_remote Y: reason" — plus the claim command built by `remote.ClaimCheckpointRemoteCommand`, for a disproved owner (a fork; rendered as information) and an unprovable one (a warning) alike. With `push_sessions` disabled both commands judge the read path instead, and also report a store the fetch side declined for a non-ownership reason. Pre-push stderr (`warnIgnoredCheckpointRemote`) is narrower: it warns only for an **unprovable** owner, because a disproved one is the ordinary fork contributor, for whom checkpoints landing in their fork is the correct outcome and a warning on every push would be noise they could silence only by claiming someone else's store. The repo is printed through `tuiutil.SanitizeDisplayText`, because it comes from the committed settings file. A local settings file present in the index or HEAD is never accepted as consent: `entire enable --local --checkpoint-remote` refuses to write it and says to untrack it, and the enable claim prompt is not offered. The enable claim prompt respects `--yes` and `--agent`, and any successful claim precedes destination resolution for the final report. It blocks nothing: checkpoints keep flowing to the elected remote, because refusing to push them turns a misconfiguration into lost work. The pre-push warning fires on **every push that delivered checkpoints** while the condition holds — a one-shot notice is seen by whoever set the repo up and not by whoever hits the problem — and is emitted after delivery, so a push carrying no checkpoints says nothing. Two gates guard it and neither is redundant: `checkpointRemoteConfigured` is the free half (an empty `checkpointURL` means "none configured" as often as "configured and refused"), and `hasCheckpointURL` is authoritative, because `PushURL` decided where this push goes while `InheritedCheckpointRemote` re-runs the vote over an identity set that can diverge from it. The claim command is offered **only for providers `parseCheckpointRemoteFlag` accepts** (github, and gitlab including nested groups): `GetCheckpointRemote` validates no provider at all, so naming any other would print a command that fails when pasted. - **Checkpoint reads follow the election** - reads consult an ordered candidate chain: the elected sync remote first, then `origin` as a **read-only legacy tier** (pre-single-remote checkpoints live there; `strategy.CheckpointReadRemotes` / `CheckpointReadRemotesWithElection` resolve the chain, failing OPEN to `[origin]` on an election error). Metadata-branch fetches refresh every candidate tracking ref because branch existence does not prove a requested checkpoint exists there; other fetch targets, tracking-ref readers, and blob hydration try candidates in order. Git-refs remote discovery ls-remotes every candidate **without** needing a dedicated `checkpoint_remote` and merges the listings. Local-ref writers stay **elected-remote-only**: `EnsurePrimaryRef`, the metadata-fetch advance, and `promoteRemoteTrackingPrimary` never act on the legacy tier (a stale origin feeding `SafelyAdvanceLocalRef` is the #1374-class hazard), keyed on the explicit election result, never on the chain's first entry. A remoteless repo keeps its "checkpoint absent" classification, which requires positive evidence (successful empty `git remote` listing, live ctx, readable settings without a `checkpoint_remote` key). From bd9cca49f1960417068f094704538e937321f626 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 14:42:25 -0700 Subject: [PATCH 02/16] fix(checkpoint): give plink -batch instead of OpenSSH -o options Under the pre-push hook, withBatchModeSSH appended `-o BatchMode=yes` to any ssh command. plink rejects `-o`, so every hook checkpoint push failed for PuTTY users. A GIT_SSH program path was also copied unquoted into GIT_SSH_COMMAND, which git runs through sh, splitting paths with spaces. Resolve the ssh variant the way git does (GIT_SSH_VARIANT, ssh.variant, then the program name parsed like split_cmdline): - OpenSSH: BatchMode + ConnectTimeout, as before. - plink/putty: -batch, plink's BatchMode; never -o. - TortoisePlink (git passes -batch itself) and simple: env left unchanged. - Unrecognized wrappers: BatchMode only, as before. Shell-quote a GIT_SSH path when rewriting it into GIT_SSH_COMMAND. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4EQFC6B4P2AY0JB87KCYFCS --- cmd/entire/cli/checkpoint/remote/git.go | 225 ++++++++++++++----- cmd/entire/cli/checkpoint/remote/git_test.go | 103 ++++++++- docs/architecture/ref-checkpoint-backend.md | 2 +- 3 files changed, 274 insertions(+), 56 deletions(-) diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index da7756e842..2faee8fd8b 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -188,22 +188,93 @@ func hasExplicitBatchMode(sshCmd string) bool { // same forms batchModeOptionRe accepts. var connectTimeoutOptionRe = regexp.MustCompile(`(?i)\bConnectTimeout\s*=\s*\S+`) +// plinkBatchArgRe matches plink's own non-interactive flag as a whole word. +var plinkBatchArgRe = regexp.MustCompile(`(?i)(^|\s)-batch(\s|$)`) + // nonInteractiveSSHConnectTimeout bounds the TCP connect to an unreachable host, // whose kernel default (~75-130s) would otherwise spend most of the pre-push // checkpoint budget on one attempt. Generous for any reachable host. const nonInteractiveSSHConnectTimeout = "30" -// isOpenSSHCommand reports whether sshCmd runs OpenSSH, by the basename of its -// first word — the same guess git makes for its ssh variant. Options are only -// added for OpenSSH: other clients (plink, custom wrappers) may reject -o. -func isOpenSSHCommand(sshCmd string) bool { - fields := strings.Fields(sshCmd) - if len(fields) == 0 { - return false +// sshVariant is the kind of ssh client git will run, which decides the options +// it accepts. Mirrors git's own variants (connect.c, determine_ssh_variant). +type sshVariant int + +const ( + sshVariantAuto sshVariant = iota // unrecognized client + sshVariantOpenSSH // takes -o options + sshVariantPlink // plink/putty: rejects -o; -batch is its BatchMode + sshVariantTortoisePlink // git passes -batch itself + sshVariantSimple // takes no options at all +) + +// overrideSSHVariant maps an explicit GIT_SSH_VARIANT / ssh.variant value the +// way git does: "auto" defers to detection, unknown values mean OpenSSH. +func overrideSSHVariant(v string) sshVariant { + switch v { + case "auto": + return sshVariantAuto + case "plink", "putty": + return sshVariantPlink + case "tortoiseplink": + return sshVariantTortoisePlink + case "simple": + return sshVariantSimple + } + return sshVariantOpenSSH +} + +// sshVariantForProgram detects the variant from a program path's basename, +// case-insensitively and with either path separator, as git does. +func sshVariantForProgram(prog string) sshVariant { + switch strings.ToLower(prog[strings.LastIndexAny(prog, `/\`)+1:]) { + case "ssh", "ssh.exe": + return sshVariantOpenSSH + case "plink", "plink.exe": + return sshVariantPlink + case "tortoiseplink", "tortoiseplink.exe": + return sshVariantTortoisePlink + } + return sshVariantAuto +} + +// firstCmdlineWord returns the program of a shell-style command line, unquoted +// the way git's split_cmdline does it: single and double quotes group, and a +// backslash outside single quotes escapes the next character. ok is false for +// an empty or unterminated command line. +func firstCmdlineWord(cmdline string) (string, bool) { + var word strings.Builder + var quote rune + started, escaped := false, false + for _, c := range strings.TrimLeftFunc(cmdline, unicode.IsSpace) { + switch { + case escaped: + word.WriteRune(c) + escaped = false + case c == '\\' && quote != '\'': + escaped, started = true, true + case quote == 0 && unicode.IsSpace(c): + return word.String(), true + case quote == 0 && (c == '\'' || c == '"'): + quote, started = c, true + case c == quote: + quote = 0 + default: + word.WriteRune(c) + started = true + } + } + if quote != 0 || escaped || !started { + return "", false } - prog := strings.Trim(fields[0], `"'`) - base := strings.ToLower(prog[strings.LastIndexAny(prog, `/\`)+1:]) - return base == "ssh" || base == "ssh.exe" + return word.String(), true +} + +// shellQuoteSSHProgram single-quotes a GIT_SSH program path for use as +// GIT_SSH_COMMAND, which git runs through sh: GIT_SSH is a bare path, so one +// containing spaces ("C:\Program Files\PuTTY\plink.exe") must stay one word. +func shellQuoteSSHProgram(path string) string { + return "'" + strings.ReplaceAll(path, "'", `'\''`) + "'" } // envLookup returns the value of the last occurrence of key in env (matching @@ -219,50 +290,110 @@ func envLookup(env []string, key string) (string, bool) { return "", false } -// gitConfigSSHCommand looks up core.sshCommand via `git config`, run with env -// so the lookup honors any HOME/GIT_CONFIG_* overrides present in env (e.g. in -// tests). Returns "" if unset or the lookup fails. -func gitConfigSSHCommand(ctx context.Context, env []string) string { - cmd := exec.CommandContext(ctx, "git", "config", "--get", "core.sshCommand") +// gitConfigSSH looks up core.sshCommand and ssh.variant in one `git config` +// call, run with env so the lookup honors any HOME/GIT_CONFIG_* overrides +// present in env (e.g. in tests). Unset values, or a failed lookup, are "". +func gitConfigSSH(ctx context.Context, env []string) (sshCommand, variant string, hasVariant bool) { + cmd := exec.CommandContext(ctx, "git", "config", "--get-regexp", `^(core\.sshcommand|ssh\.variant)$`) cmd.Env = env out, err := cmd.Output() if err != nil { - return "" + return "", "", false + } + // Last value wins, as for git's own single-valued lookups. + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + key, value, _ := strings.Cut(line, " ") + switch key { + case "core.sshcommand": + sshCommand = strings.TrimSpace(value) + case "ssh.variant": + variant, hasVariant = value, true + } } - return strings.TrimSpace(string(out)) + return sshCommand, variant, hasVariant } -// effectiveSSHCommand resolves the ssh invocation git itself would use, in -// git's own precedence order: the GIT_SSH_COMMAND environment variable, then -// the core.sshCommand git config value, then the GIT_SSH environment -// variable, falling back to plain "ssh" when none are set. -func effectiveSSHCommand(ctx context.Context, env []string) string { - if v, ok := envLookup(env, "GIT_SSH_COMMAND"); ok { - if trimmed := strings.TrimSpace(v); trimmed != "" { - return trimmed - } - } - if v := gitConfigSSHCommand(ctx, env); v != "" { - return v - } - if v, ok := envLookup(env, "GIT_SSH"); ok { - if trimmed := strings.TrimSpace(v); trimmed != "" { - return trimmed +// resolvedSSH is the ssh client git itself would run for a remote operation. +type resolvedSSH struct { + command string // shell command line (or bare program path when !cmdline) + cmdline bool // false when command came from GIT_SSH, a bare program path + variant sshVariant +} + +// resolveSSH resolves the ssh invocation git itself would use, in git's own +// precedence order: the GIT_SSH_COMMAND environment variable, then the +// core.sshCommand git config value, then the GIT_SSH environment variable, +// falling back to plain "ssh" when none are set. The variant comes from +// GIT_SSH_VARIANT or ssh.variant when set, else from the program's name. +func resolveSSH(ctx context.Context, env []string) resolvedSSH { + configCmd, configVariant, hasConfigVariant := gitConfigSSH(ctx, env) + res := resolvedSSH{command: "ssh", cmdline: true} + if v, ok := envLookup(env, "GIT_SSH_COMMAND"); ok && strings.TrimSpace(v) != "" { + res.command = strings.TrimSpace(v) + } else if configCmd != "" { + res.command = configCmd + } else if v, ok := envLookup(env, "GIT_SSH"); ok && strings.TrimSpace(v) != "" { + res.command, res.cmdline = strings.TrimSpace(v), false + } + + if v, ok := envLookup(env, "GIT_SSH_VARIANT"); ok { + res.variant = overrideSSHVariant(v) + return res + } + if hasConfigVariant { + res.variant = overrideSSHVariant(configVariant) + return res + } + prog := res.command + if res.cmdline { + word, ok := firstCmdlineWord(res.command) + if !ok { + return res // sshVariantAuto } + prog = word } - return "ssh" + res.variant = sshVariantForProgram(prog) + return res } -// withBatchModeSSH returns env with GIT_SSH_COMMAND set so ssh runs with -// BatchMode=yes and, for OpenSSH, a ConnectTimeout. The base ssh invocation is resolved via effectiveSSHCommand -// (env GIT_SSH_COMMAND > core.sshCommand > GIT_SSH > plain "ssh") so a custom -// ssh command configured via core.sshCommand isn't silently discarded. The -// flag is only appended when BatchMode isn't already explicitly set — an -// existing BatchMode=no is a deliberate user choice and is left untouched — -// so the result is idempotent. +// withBatchModeSSH returns env with GIT_SSH_COMMAND set so the ssh client git +// runs cannot prompt: OpenSSH and unrecognized clients get BatchMode=yes, plink +// gets its equivalent -batch, and OpenSSH also gets a ConnectTimeout. The base +// invocation is resolved via resolveSSH, so a custom ssh command configured via +// core.sshCommand or GIT_SSH isn't silently discarded. Options a client would +// reject are never added: plink fails on -o, and TortoisePlink and the +// "simple" variant need nothing from us. An option already set explicitly is +// left as the user chose it — an existing BatchMode=no is deliberate — so the +// result is idempotent. env is returned unchanged when nothing needs adding. func withBatchModeSSH(ctx context.Context, env []string) []string { const key = "GIT_SSH_COMMAND=" - base := effectiveSSHCommand(ctx, env) + ssh := resolveSSH(ctx, env) + var opts string + switch ssh.variant { + case sshVariantOpenSSH, sshVariantAuto: + if !hasExplicitBatchMode(ssh.command) { + opts += " -o BatchMode=yes" + } + // Command-line -o beats ~/.ssh/config, so a ConnectTimeout set there is + // overridden here; one set on the command itself is left alone. + // Unrecognized clients get BatchMode only, as before: wrappers that + // exec ssh "$@" need it, but a timeout is not ours to impose on them. + if ssh.variant == sshVariantOpenSSH && !connectTimeoutOptionRe.MatchString(ssh.command) { + opts += " -o ConnectTimeout=" + nonInteractiveSSHConnectTimeout + } + case sshVariantPlink: + if !plinkBatchArgRe.MatchString(ssh.command) { + opts += " -batch" + } + case sshVariantTortoisePlink, sshVariantSimple: + } + if opts == "" { + return env + } + base := ssh.command + if !ssh.cmdline { + base = shellQuoteSSHProgram(base) + } out := make([]string, 0, len(env)+1) for _, e := range env { if strings.HasPrefix(e, key) { @@ -270,15 +401,7 @@ func withBatchModeSSH(ctx context.Context, env []string) []string { } out = append(out, e) } - if !hasExplicitBatchMode(base) { - base += " -o BatchMode=yes" - } - // Command-line -o beats ~/.ssh/config, so a ConnectTimeout set there is - // overridden here; one set on the command itself is left alone. - if isOpenSSHCommand(base) && !connectTimeoutOptionRe.MatchString(base) { - base += " -o ConnectTimeout=" + nonInteractiveSSHConnectTimeout - } - return append(out, key+base) + return append(out, key+base+opts) } // applyNonInteractiveSSH sets BatchMode SSH on cmd when ctx is marked diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index 6014efc2c6..7bb2196d7d 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -1170,7 +1170,7 @@ func TestWithBatchModeSSH(t *testing.T) { in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH=/usr/local/bin/custom-ssh") }, - want: "/usr/local/bin/custom-ssh -o BatchMode=yes", + want: "'/usr/local/bin/custom-ssh' -o BatchMode=yes", }, { name: "unrelated substring containing BatchMode-like text does not count as explicit", @@ -1194,25 +1194,120 @@ func TestWithBatchModeSSH(t *testing.T) { want: "/usr/bin/ssh -i key -o BatchMode=yes -o ConnectTimeout=30", }, { - name: "non-OpenSSH client gets no ConnectTimeout", + name: "plink that already runs -batch is left untouched", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=plink -batch") }, - want: "plink -batch -o BatchMode=yes", + want: "plink -batch", + }, + { + name: "plink gets -batch, not -o options it rejects", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=plink.exe -i key.ppk") + }, + want: "plink.exe -i key.ppk -batch", + }, + { + name: "single-quoted plink path with spaces is recognized", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, `GIT_SSH_COMMAND='C:\Program Files\PuTTY\plink.exe'`) + }, + want: `'C:\Program Files\PuTTY\plink.exe' -batch`, + }, + { + name: "GIT_SSH plink path with spaces is quoted as one word", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, `GIT_SSH=C:\Program Files\PuTTY\PLINK.EXE`) + }, + want: `'C:\Program Files\PuTTY\PLINK.EXE' -batch`, + }, + { + name: "GIT_SSH TortoisePlink needs nothing: git passes -batch itself", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, `GIT_SSH=C:\TortoiseGit\bin\TortoisePlink.exe`) + }, + want: "", + }, + { + name: "GIT_SSH_VARIANT=plink declares a wrapper as plink", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=my-wrapper", "GIT_SSH_VARIANT=plink") + }, + want: "my-wrapper -batch", + }, + { + name: "GIT_SSH_VARIANT=ssh declares a wrapper as OpenSSH", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=my-wrapper", "GIT_SSH_VARIANT=ssh") + }, + want: "my-wrapper -o BatchMode=yes -o ConnectTimeout=30", + }, + { + name: "ssh.variant=simple config takes no options", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=my-wrapper", rawGitConfigFile(t, "[ssh]\n\tvariant = simple\n")) + }, + want: "my-wrapper", + }, + { + name: "GIT_SSH_VARIANT env beats ssh.variant config", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=my-wrapper", "GIT_SSH_VARIANT=plink", + rawGitConfigFile(t, "[ssh]\n\tvariant = simple\n")) + }, + want: "my-wrapper -batch", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - out := withBatchModeSSH(context.Background(), tt.in(t)) + in := tt.in(t) + out := withBatchModeSSH(context.Background(), in) got, ok := envToMap(out)["GIT_SSH_COMMAND"] + if tt.want == "" { + assert.False(t, ok, "GIT_SSH_COMMAND should not be set") + assert.Equal(t, in, out, "env should be returned unchanged") + return + } assert.True(t, ok, "GIT_SSH_COMMAND should be set") assert.Equal(t, tt.want, got) }) } } +// rawGitConfigFile writes content as a global gitconfig and returns the +// GIT_CONFIG_GLOBAL env entry pointing at it. +func rawGitConfigFile(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "gitconfig") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + return "GIT_CONFIG_GLOBAL=" + path +} + +func TestFirstCmdlineWord(t *testing.T) { + t.Parallel() + tests := []struct { + in string + want string + ok bool + }{ + {in: "ssh -i key", want: "ssh", ok: true}, + {in: " plink", want: "plink", ok: true}, + {in: `'C:\Program Files\plink.exe' -batch`, want: `C:\Program Files\plink.exe`, ok: true}, + // As in git's split_cmdline, a backslash escapes outside single quotes. + {in: `"C:\Program Files\plink.exe"`, want: `C:Program Filesplink.exe`, ok: true}, + {in: `/opt/my\ ssh -v`, want: `/opt/my ssh`, ok: true}, + {in: `'unterminated`, ok: false}, + {in: " ", ok: false}, + } + for _, tt := range tests { + got, ok := firstCmdlineWord(tt.in) + assert.Equal(t, tt.ok, ok, tt.in) + assert.Equal(t, tt.want, got, tt.in) + } +} + func TestUnreachableRemoteLine(t *testing.T) { t.Parallel() pushErr := func(output string) error { diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index 5074796837..2186b83b7a 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -118,7 +118,7 @@ without additional secret redaction of the remote's diagnostic output. Plain non SSH authentication failures retain their dedicated hint. Failures remain queued and **never fail the user's git push**. -The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch that spends the budget leaves the fallback none, so its one guaranteed attempt fails at once and the refs stay queued. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it. Under the pre-push hook, OpenSSH also gets `ConnectTimeout` (`withBatchModeSSH`), so an unreachable host fails within seconds instead of spending the budget on a kernel connect timeout. +The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch that spends the budget leaves the fallback none, so its one guaranteed attempt fails at once and the refs stay queued. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it. Under the pre-push hook, OpenSSH also gets `ConnectTimeout` (`withBatchModeSSH`), so an unreachable host fails within seconds instead of spending the budget on a kernel connect timeout. Other ssh clients get only what they accept, mirroring git's own variant detection (`GIT_SSH_VARIANT`, `ssh.variant`, then the program name): plink gets `-batch`, its equivalent of `BatchMode`, never `-o`; TortoisePlink and the `simple` variant get nothing; an unrecognized wrapper keeps `BatchMode` only. A `GIT_SSH` program path is shell-quoted when rewritten into `GIT_SSH_COMMAND`, so a path with spaces stays one word. A remote that cannot be reached at all ends the flush after one attempt. When nothing has landed and a chunk's git output carries a connect-phase failure from ssh or curl (`remote.UnreachableRemoteLine`: unresolvable host, failed connect), later chunks and the per-ref fallback would each fail the same way after their own connect timeout, so the flush stops, prints that one line — credentials redacted, control bytes stripped, capped — and leaves the whole queue in order. The match is deliberately narrow: no bare strerror text (curl's is localized), no mid-transfer timeouts (the host was reachable), and no `remote:` lines (a reachable remote's own hook failing to resolve a host). A miss only costs the old path, the bounded fallback. From 8af0e8610e70ae67becccdff75b67b1bb571c67f Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 16:48:37 -0700 Subject: [PATCH 03/16] fix(checkpoint): adapt chunk size to the link and harden ssh handling Review fixes for the bounded git-refs pre-push flush: - A link too slow to land one chunk within the flush budget cut the same head-of-queue chunk on every push and never made progress. The chunk size now adapts (PushQueue.ChunkSizeHint): halved when the budget cuts a chunk, doubled back after a clean batch. A budget cut ends the flush ("Stopped pushing") instead of running the per-ref fallback on a spent budget and reporting it as a failed retry, and rotates the cut chunk to the back. - The migration push's unbounded batch no longer leaves its per-ref fallback an already-spent budget. - ssh.variant / GIT_SSH_VARIANT "auto" defers to program-name detection, as in git, so plink behind it gets -batch rather than -o. - An unreadable git config leaves the environment unchanged instead of replacing the user's core.sshCommand with plain ssh. - ConnectTimeout raised to 60s for slow-starting ssh proxies. - Credential redaction covers an unencoded "@" in a password; the printed cause line drops bidi controls; curl's "failed to connect to" matches only in its " port" form. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4EYPFCATE1W2XQ7GKS7HBH2 --- cmd/entire/cli/checkpoint/pushqueue.go | 36 +++++ cmd/entire/cli/checkpoint/remote/git.go | 61 ++++++--- cmd/entire/cli/checkpoint/remote/git_test.go | 53 ++++++-- cmd/entire/cli/gitremote/gitremote.go | 4 +- cmd/entire/cli/gitremote/gitremote_test.go | 16 +++ cmd/entire/cli/strategy/manual_commit_push.go | 128 +++++++++++++----- cmd/entire/cli/strategy/push_common.go | 19 ++- .../cli/strategy/refs_push_bounds_test.go | 79 ++++++++++- docs/architecture/ref-checkpoint-backend.md | 2 +- docs/development/checkpoint-implementation.md | 2 +- 10 files changed, 323 insertions(+), 77 deletions(-) diff --git a/cmd/entire/cli/checkpoint/pushqueue.go b/cmd/entire/cli/checkpoint/pushqueue.go index b60ae97f6f..c06a2553ac 100644 --- a/cmd/entire/cli/checkpoint/pushqueue.go +++ b/cmd/entire/cli/checkpoint/pushqueue.go @@ -8,6 +8,8 @@ import ( "fmt" "os" "path/filepath" + "strconv" + "strings" "github.com/go-git/go-git/v6" "github.com/go-git/go-git/v6/plumbing" @@ -25,6 +27,9 @@ import ( const ( pushQueueFileName = "entire-checkpoint-push-queue.jsonl" pushQueueLockName = "entire-checkpoint-push-queue.lock" + // pushChunkSizeFileName remembers how many refs one batch push can carry + // within the flush budget on this machine's link (see ChunkSizeHint). + pushChunkSizeFileName = "entire-checkpoint-push-chunk-size" ) // lock opens the common dir's root and takes the queue lock inside it, @@ -314,3 +319,34 @@ func writeQueueAtomic(root *os.Root, data []byte) error { } return nil } + +// ChunkSizeHint returns the remembered batch-push chunk size, clamped to +// [1, maxSize], or maxSize when none is remembered or it cannot be read. +func (q *PushQueue) ChunkSizeHint(maxSize int) int { + root, err := q.root() + if err != nil { + return maxSize + } + data, err := root.ReadFile(pushChunkSizeFileName) + if err != nil { + return maxSize + } + n, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil || n < 1 || n > maxSize { + return maxSize + } + return n +} + +// SetChunkSizeHint remembers n as the batch-push chunk size. Best-effort: a +// failure only means the next flush starts from the default again. +func (q *PushQueue) SetChunkSizeHint(n int) error { + root, err := q.root() + if err != nil { + return fmt.Errorf("open git common dir: %w", err) + } + if err := jsonutil.WriteFileAtomicIn(root, pushChunkSizeFileName, []byte(strconv.Itoa(n)+"\n"), 0o600); err != nil { + return fmt.Errorf("write push chunk size: %w", err) + } + return nil +} diff --git a/cmd/entire/cli/checkpoint/remote/git.go b/cmd/entire/cli/checkpoint/remote/git.go index 2faee8fd8b..1997ffb740 100644 --- a/cmd/entire/cli/checkpoint/remote/git.go +++ b/cmd/entire/cli/checkpoint/remote/git.go @@ -117,11 +117,13 @@ var unreachableRemoteNeedles = []string{ "ssh: could not resolve hostname", "could not resolve host:", "could not resolve proxy:", - "failed to connect to ", + curlFailedToConnect, // only in curl's " port " form, not a helper's "couldn't connect to server", "resolving timed out", } +const curlFailedToConnect = "failed to connect to " + // maxUnreachableLineRunes caps the cause line printed inside the user's push. const maxUnreachableLineRunes = 200 @@ -146,7 +148,7 @@ func UnreachableRemoteLine(err error) (string, bool) { } lower := strings.ToLower(line) for _, n := range unreachableRemoteNeedles { - if strings.Contains(lower, n) { + if strings.Contains(lower, n) && (n != curlFailedToConnect || strings.Contains(lower, " port ")) { return displayGitLine(line), true } } @@ -160,7 +162,9 @@ func displayGitLine(line string) string { line = strings.TrimPrefix(line, "fatal: ") line = gitremote.RedactCredentialsInText(line) line = strings.Map(func(r rune) rune { - if unicode.IsControl(r) { + // Cc: terminal escapes and newlines; Cf: bidi overrides that could + // make the line read as something else. + if unicode.IsControl(r) || unicode.Is(unicode.Cf, r) { return -1 } return r @@ -193,8 +197,10 @@ var plinkBatchArgRe = regexp.MustCompile(`(?i)(^|\s)-batch(\s|$)`) // nonInteractiveSSHConnectTimeout bounds the TCP connect to an unreachable host, // whose kernel default (~75-130s) would otherwise spend most of the pre-push -// checkpoint budget on one attempt. Generous for any reachable host. -const nonInteractiveSSHConnectTimeout = "30" +// checkpoint budget on one attempt. Generous for any reachable host, including +// one behind a slow-starting ProxyCommand (SSM, cloudflared, Teleport), whose +// banner wait ssh's ConnectTimeout also bounds. +const nonInteractiveSSHConnectTimeout = "60" // sshVariant is the kind of ssh client git will run, which decides the options // it accepts. Mirrors git's own variants (connect.c, determine_ssh_variant). @@ -292,13 +298,16 @@ func envLookup(env []string, key string) (string, bool) { // gitConfigSSH looks up core.sshCommand and ssh.variant in one `git config` // call, run with env so the lookup honors any HOME/GIT_CONFIG_* overrides -// present in env (e.g. in tests). Unset values, or a failed lookup, are "". -func gitConfigSSH(ctx context.Context, env []string) (sshCommand, variant string, hasVariant bool) { +// present in env (e.g. in tests). Unset values are "". ok is false when the +// lookup itself failed, as opposed to finding nothing (git's exit status 1): +// the caller must not then guess at the command git will run. +func gitConfigSSH(ctx context.Context, env []string) (sshCommand, variant string, hasVariant, ok bool) { cmd := exec.CommandContext(ctx, "git", "config", "--get-regexp", `^(core\.sshcommand|ssh\.variant)$`) cmd.Env = env out, err := cmd.Output() if err != nil { - return "", "", false + var exitErr *exec.ExitError + return "", "", false, errors.As(err, &exitErr) && exitErr.ExitCode() == 1 } // Last value wins, as for git's own single-valued lookups. for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { @@ -310,7 +319,7 @@ func gitConfigSSH(ctx context.Context, env []string) (sshCommand, variant string variant, hasVariant = value, true } } - return sshCommand, variant, hasVariant + return sshCommand, variant, hasVariant, true } // resolvedSSH is the ssh client git itself would run for a remote operation. @@ -324,36 +333,43 @@ type resolvedSSH struct { // precedence order: the GIT_SSH_COMMAND environment variable, then the // core.sshCommand git config value, then the GIT_SSH environment variable, // falling back to plain "ssh" when none are set. The variant comes from -// GIT_SSH_VARIANT or ssh.variant when set, else from the program's name. -func resolveSSH(ctx context.Context, env []string) resolvedSSH { - configCmd, configVariant, hasConfigVariant := gitConfigSSH(ctx, env) +// GIT_SSH_VARIANT or ssh.variant when set, else from the program's name. ok is +// false when the command would come from git config that could not be read: +// the caller then cannot know what git will run, and must not replace it. +func resolveSSH(ctx context.Context, env []string) (resolvedSSH, bool) { + configCmd, configVariant, hasConfigVariant, configOK := gitConfigSSH(ctx, env) res := resolvedSSH{command: "ssh", cmdline: true} if v, ok := envLookup(env, "GIT_SSH_COMMAND"); ok && strings.TrimSpace(v) != "" { res.command = strings.TrimSpace(v) + } else if !configOK { + return res, false } else if configCmd != "" { res.command = configCmd } else if v, ok := envLookup(env, "GIT_SSH"); ok && strings.TrimSpace(v) != "" { res.command, res.cmdline = strings.TrimSpace(v), false } + // An explicit variant decides, except "auto", which (as in git) means + // detect from the program name below. if v, ok := envLookup(env, "GIT_SSH_VARIANT"); ok { - res.variant = overrideSSHVariant(v) - return res - } - if hasConfigVariant { - res.variant = overrideSSHVariant(configVariant) - return res + if res.variant = overrideSSHVariant(v); res.variant != sshVariantAuto { + return res, true + } + } else if hasConfigVariant { + if res.variant = overrideSSHVariant(configVariant); res.variant != sshVariantAuto { + return res, true + } } prog := res.command if res.cmdline { word, ok := firstCmdlineWord(res.command) if !ok { - return res // sshVariantAuto + return res, true // sshVariantAuto } prog = word } res.variant = sshVariantForProgram(prog) - return res + return res, true } // withBatchModeSSH returns env with GIT_SSH_COMMAND set so the ssh client git @@ -367,7 +383,10 @@ func resolveSSH(ctx context.Context, env []string) resolvedSSH { // result is idempotent. env is returned unchanged when nothing needs adding. func withBatchModeSSH(ctx context.Context, env []string) []string { const key = "GIT_SSH_COMMAND=" - ssh := resolveSSH(ctx, env) + ssh, ok := resolveSSH(ctx, env) + if !ok { + return env + } var opts string switch ssh.variant { case sshVariantOpenSSH, sshVariantAuto: diff --git a/cmd/entire/cli/checkpoint/remote/git_test.go b/cmd/entire/cli/checkpoint/remote/git_test.go index 7bb2196d7d..b9c11dd451 100644 --- a/cmd/entire/cli/checkpoint/remote/git_test.go +++ b/cmd/entire/cli/checkpoint/remote/git_test.go @@ -1119,35 +1119,35 @@ func TestWithBatchModeSSH(t *testing.T) { { name: "no existing GIT_SSH_COMMAND or config defaults to ssh", in: func(t *testing.T) []string { return isolatedSSHEnv(t) }, - want: "ssh -o BatchMode=yes -o ConnectTimeout=30", + want: "ssh -o BatchMode=yes -o ConnectTimeout=60", }, { name: "preserves and extends a custom ssh command", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=ssh -i /home/me/.ssh/id") }, - want: "ssh -i /home/me/.ssh/id -o BatchMode=yes -o ConnectTimeout=30", + want: "ssh -i /home/me/.ssh/id -o BatchMode=yes -o ConnectTimeout=60", }, { name: "GIT_SSH_COMMAND with explicit BatchMode=yes is left untouched", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=ssh -o BatchMode=yes") }, - want: "ssh -o BatchMode=yes -o ConnectTimeout=30", + want: "ssh -o BatchMode=yes -o ConnectTimeout=60", }, { name: "GIT_SSH_COMMAND with explicit BatchMode=no is respected, not overridden", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=ssh -o BatchMode=no") }, - want: "ssh -o BatchMode=no -o ConnectTimeout=30", + want: "ssh -o BatchMode=no -o ConnectTimeout=60", }, { name: "blank GIT_SSH_COMMAND falls back to ssh", in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND= ") }, - want: "ssh -o BatchMode=yes -o ConnectTimeout=30", + want: "ssh -o BatchMode=yes -o ConnectTimeout=60", }, { name: "core.sshCommand git config is used as the base when env is unset", @@ -1155,7 +1155,7 @@ func TestWithBatchModeSSH(t *testing.T) { cfg := gitConfigFile(t, "ssh -i /home/me/.ssh/work_key") return isolatedSSHEnv(t, cfg) }, - want: "ssh -i /home/me/.ssh/work_key -o BatchMode=yes -o ConnectTimeout=30", + want: "ssh -i /home/me/.ssh/work_key -o BatchMode=yes -o ConnectTimeout=60", }, { name: "GIT_SSH_COMMAND env takes precedence over core.sshCommand config", @@ -1163,7 +1163,7 @@ func TestWithBatchModeSSH(t *testing.T) { cfg := gitConfigFile(t, "ssh -i /home/me/.ssh/work_key") return isolatedSSHEnv(t, cfg, "GIT_SSH_COMMAND=ssh -i /home/me/.ssh/personal_key") }, - want: "ssh -i /home/me/.ssh/personal_key -o BatchMode=yes -o ConnectTimeout=30", + want: "ssh -i /home/me/.ssh/personal_key -o BatchMode=yes -o ConnectTimeout=60", }, { name: "GIT_SSH is used only when neither env GIT_SSH_COMMAND nor config are set", @@ -1177,7 +1177,7 @@ func TestWithBatchModeSSH(t *testing.T) { in: func(t *testing.T) []string { return isolatedSSHEnv(t, `GIT_SSH_COMMAND=ssh -o ProxyCommand="connect -H proxy NoBatchModeHereEither"`) }, - want: `ssh -o ProxyCommand="connect -H proxy NoBatchModeHereEither" -o BatchMode=yes -o ConnectTimeout=30`, + want: `ssh -o ProxyCommand="connect -H proxy NoBatchModeHereEither" -o BatchMode=yes -o ConnectTimeout=60`, }, { name: "explicit ConnectTimeout is left untouched", @@ -1191,7 +1191,7 @@ func TestWithBatchModeSSH(t *testing.T) { in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=/usr/bin/ssh -i key") }, - want: "/usr/bin/ssh -i key -o BatchMode=yes -o ConnectTimeout=30", + want: "/usr/bin/ssh -i key -o BatchMode=yes -o ConnectTimeout=60", }, { name: "plink that already runs -batch is left untouched", @@ -1240,7 +1240,7 @@ func TestWithBatchModeSSH(t *testing.T) { in: func(t *testing.T) []string { return isolatedSSHEnv(t, "GIT_SSH_COMMAND=my-wrapper", "GIT_SSH_VARIANT=ssh") }, - want: "my-wrapper -o BatchMode=yes -o ConnectTimeout=30", + want: "my-wrapper -o BatchMode=yes -o ConnectTimeout=60", }, { name: "ssh.variant=simple config takes no options", @@ -1249,6 +1249,27 @@ func TestWithBatchModeSSH(t *testing.T) { }, want: "my-wrapper", }, + { + name: "variant auto still detects plink from the program name", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, `GIT_SSH=C:\\PuTTY\\plink.exe`, rawGitConfigFile(t, "[ssh]\n\tvariant = auto\n")) + }, + want: `'C:\\PuTTY\\plink.exe' -batch`, + }, + { + name: "GIT_SSH_VARIANT=auto still gives OpenSSH its timeout", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, "GIT_SSH_COMMAND=ssh", "GIT_SSH_VARIANT=auto") + }, + want: "ssh -o BatchMode=yes -o ConnectTimeout=60", + }, + { + name: "unreadable git config leaves the user's ssh command alone", + in: func(t *testing.T) []string { + return isolatedSSHEnv(t, rawGitConfigFile(t, "[core\n\tsshCommand = my-ssh\n")) + }, + want: "", + }, { name: "GIT_SSH_VARIANT env beats ssh.variant config", in: func(t *testing.T) []string { @@ -1367,6 +1388,16 @@ func TestUnreachableRemoteLine(t *testing.T) { name: "non-fast-forward rejection", err: pushErr(" ! [rejected] refs/x -> refs/x (non-fast-forward)\nerror: failed to push some refs"), }, + { + name: "bidi overrides are stripped from the printed line", + err: pushErr("ssh: connect to host gith\u202eub.com port 22: Connection refused"), + want: "ssh: connect to host github.com port 22: Connection refused", + ok: true, + }, + { + name: "a helper's own connect failure is not curl's", + err: pushErr("failed to connect to keyring: no such service\nfatal: Authentication failed"), + }, {name: "nil", err: nil}, } for _, tt := range tests { @@ -1396,7 +1427,7 @@ func TestWithBatchModeSSH_PreservesOtherVarsWithoutDuplicating(t *testing.T) { m := envToMap(out) assert.Equal(t, "value", m["SOME_OTHER_VAR"]) - assert.Equal(t, "ssh -o BatchMode=yes -o ConnectTimeout=30", m["GIT_SSH_COMMAND"]) + assert.Equal(t, "ssh -o BatchMode=yes -o ConnectTimeout=60", m["GIT_SSH_COMMAND"]) } // TestNewCommand_NonInteractiveSSH verifies that a checkpoint git command built diff --git a/cmd/entire/cli/gitremote/gitremote.go b/cmd/entire/cli/gitremote/gitremote.go index 1a7164a3a8..26ba10f9d8 100644 --- a/cmd/entire/cli/gitremote/gitremote.go +++ b/cmd/entire/cli/gitremote/gitremote.go @@ -385,7 +385,9 @@ func RedactURL(rawURL string) string { } // credentialInURL matches the scheme and userinfo of a URL embedded in text. -var credentialInURL = regexp.MustCompile(`([a-zA-Z][a-zA-Z0-9+.-]*://)[^/@\s]*@`) +// Greedy up to the last "@" before the path, so an unencoded "@" in a password +// cannot leave part of it behind. +var credentialInURL = regexp.MustCompile(`([a-zA-Z][a-zA-Z0-9+.-]*://)[^/\s]*@`) // RedactCredentialsInText removes userinfo from any URL appearing in free text, // for git's output where we do not know which URL was echoed. diff --git a/cmd/entire/cli/gitremote/gitremote_test.go b/cmd/entire/cli/gitremote/gitremote_test.go index 16ba487b17..e20967c8a5 100644 --- a/cmd/entire/cli/gitremote/gitremote_test.go +++ b/cmd/entire/cli/gitremote/gitremote_test.go @@ -496,3 +496,19 @@ func TestParseURL_TrailingSeparatorsAreStrippedBeforeTheSuffix(t *testing.T) { }) } } + +func TestRedactCredentialsInText(t *testing.T) { + t.Parallel() + tests := []struct{ in, want string }{ + {"fatal: unable to access 'https://bob:hunter2@git.example.com/x/': nope", "fatal: unable to access 'https://git.example.com/x/': nope"}, + // An unencoded "@" in the password must not leave its tail behind. + {"https://u:p@ss@host/repo", "https://host/repo"}, + {"ssh://git@github.com/o/r and https://t@h.example/x", "ssh://github.com/o/r and https://h.example/x"}, + {"no url here", "no url here"}, + } + for _, tt := range tests { + if got := RedactCredentialsInText(tt.in); got != tt.want { + t.Errorf("RedactCredentialsInText(%q) = %q; want %q", tt.in, got, tt.want) + } + } +} diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index 200376831c..be70d6b2c5 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -473,8 +473,10 @@ func PushQueuedCheckpointRefs(ctx context.Context, repo *git.Repository, remote // Shared by the git-refs pre-push path (which logs and ignores the error to // never block the user's push) and the migration command's opt-in push (which // surfaces it). boundBatch puts the batch push under checkpointFlushBudget too; -// the pre-push path sets it, the explicit migration push does not. Stale entries — refs no longer present locally — are pruned so -// they don't block the queue forever. +// the pre-push path sets it, the explicit migration push does not, and its +// per-ref fallback then gets a fresh budget after the batch. Stale entries — +// refs no longer present locally — are pruned so they don't block the queue +// forever. func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps pushSettings, boundBatch bool) (int, error) { queue, err := checkpoint.PushQueueForRepo(ctx, repo) if err != nil { @@ -517,21 +519,30 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push // One budget for the whole flush, opened before the batch: a backlog upload // is as able to hang the user's push as a walk of per-ref retries. Bounded; - // see checkpointFlushBudget and maxConsecutiveRefPushFailures. + // see checkpointFlushBudget and maxConsecutiveRefPushFailures. An unbounded + // batch opens it after the batch instead, so its fallback is not left with + // a budget the batch already spent. flushCtx, cancelFlush := context.WithTimeout(pushCtx, checkpointFlushBudget) - defer cancelFlush() batchCtx := flushCtx if !boundBatch { + cancelFlush() batchCtx = pushCtx } // Fast path: push the refs a chunk per round-trip (fast-forward-only). - batch := pushRefChunks(batchCtx, dest.target, existing, func(landed []plumbing.ReferenceName) { + chunkSize := queue.ChunkSizeHint(checkpointRefPushChunkSize) + batch := pushRefChunks(batchCtx, dest.target, existing, chunkSize, func(landed []plumbing.ReferenceName) { if removeErr := queue.Remove(landed); removeErr != nil { logging.Warn(ctx, "git-refs push: clear pushed refs from queue failed", slog.String("error", removeErr.Error())) } }) + if !boundBatch { + flushCtx, cancelFlush = context.WithTimeout(pushCtx, checkpointFlushBudget) + } + defer cancelFlush() + budgetCut := errors.Is(flushCtx.Err(), context.DeadlineExceeded) && pushCtx.Err() == nil + adaptChunkSize(ctx, queue, chunkSize, batch, budgetCut) if len(batch.failed) == 0 && len(batch.untried) == 0 { stop(" done") return batch.landed, nil @@ -547,39 +558,20 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push slog.Int("queued", len(existing))) return batch.landed, nil } + // The budget ran out mid-chunk: the per-ref fallback would only fail its + // one guaranteed attempt on the spent budget, reporting a retry that never + // ran as a push failure. It is the same stop as running out between chunks. + if budgetCut && !batch.sshAuthFailed && batch.unreachable == "" { + stopOnBudgetCut(ctx, flushCtx, queue, batch, len(existing), stop) + return batch.landed, nil + } if batch.unreachable != "" { stop(" failed") } else { stop("") } - // Non-interactive SSH auth failures cannot be fixed by per-ref - // fetch+replay. Surface the same actionable hint as the v1 doPushRef path - // (issue #1523) instead of only logging to .entire/logs/. - // Deliberately does not print the batch error: it carries git's own output, - // and this runs inside the user's `git push`. The hint below is the - // actionable part; the full error reaches .entire/logs via the caller, which - // logs the error this branch returns. - if batch.sshAuthFailed { - fmt.Fprintln(os.Stderr, "[entire] Warning: couldn't push checkpoint refs (SSH authentication failed).") - printNonInteractiveSSHAuthHint() - if dest.checkpointRemote { - printCheckpointRemoteHint(dest.target) - } - return batch.landed, batch.firstErr - } - - // Nothing landed and git could not even connect: the per-ref fallback would - // fail the same way, one connect timeout per ref. Unlike the branches above, - // name the cause: it is one line from ssh or curl (credentials redacted, see - // remote.UnreachableRemoteLine), and without it the user sees a slow push - // fail with no reason. No rotation: no ref failed on its own account. - if batch.unreachable != "" { - fmt.Fprintf(os.Stderr, "[entire] Couldn't reach %s: %s\n", dest.display(), batch.unreachable) - fmt.Fprintf(os.Stderr, "[entire] %d checkpoint ref(s) stay queued for the next push.\n", len(existing)) - if dest.checkpointRemote { - printCheckpointRemoteHint(dest.target) - } + if reportUnpushableDestination(dest, batch, len(existing)) { return batch.landed, batch.firstErr } @@ -659,6 +651,78 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push return totalPushed, nil } +// reportUnpushableDestination prints the hint for a batch that failed because +// the destination refused the SSH key or could not be reached, and reports +// whether it was one of those: the per-ref fallback cannot fix either. +func reportUnpushableDestination(dest refsPushDestination, batch chunkPushResult, queued int) bool { + // Non-interactive SSH auth failures cannot be fixed by per-ref + // fetch+replay. Surface the same actionable hint as the v1 doPushRef path + // (issue #1523) instead of only logging to .entire/logs/. + // Deliberately does not print the batch error: it carries git's own output, + // and this runs inside the user's `git push`. The hint below is the + // actionable part; the full error reaches .entire/logs via the caller, which + // logs the error this branch returns. + if batch.sshAuthFailed { + fmt.Fprintln(os.Stderr, "[entire] Warning: couldn't push checkpoint refs (SSH authentication failed).") + printNonInteractiveSSHAuthHint() + if dest.checkpointRemote { + printCheckpointRemoteHint(dest.target) + } + return true + } + + // Nothing landed and git could not even connect: the per-ref fallback would + // fail the same way, one connect timeout per ref. Unlike the branches above, + // name the cause: it is one line from ssh or curl (credentials redacted, see + // remote.UnreachableRemoteLine), and without it the user sees a slow push + // fail with no reason. No rotation: no ref failed on its own account. + if batch.unreachable != "" { + fmt.Fprintf(os.Stderr, "[entire] Couldn't reach %s: %s\n", dest.display(), batch.unreachable) + fmt.Fprintf(os.Stderr, "[entire] %d checkpoint ref(s) stay queued for the next push.\n", queued) + if dest.checkpointRemote { + printCheckpointRemoteHint(dest.target) + } + return true + } + return false +} + +// stopOnBudgetCut reports and settles a batch the flush budget cut mid-chunk. +func stopOnBudgetCut(ctx, flushCtx context.Context, queue *checkpoint.PushQueue, batch chunkPushResult, queued int, stop func(string)) { + stop(fmt.Sprintf(" pushed %d of %d", batch.landed, queued)) + fmt.Fprintf(os.Stderr, "[entire] Stopped pushing: %s; %d checkpoint ref(s) stay queued for the next push.\n", + flushAbortReason(flushCtx, 0), queued-batch.landed) + logging.Warn(ctx, "git-refs push: batch push cut by the flush budget; remaining refs stay queued", + slog.Int("pushed", batch.landed), slog.Int("queued", queued)) + // Fair scheduling, as after a fallback abort: a slow remote cuts the + // budget at the same place every push, so the cut chunk goes to the + // back rather than holding the head of the queue. + if err := queue.Rotate(batch.failed); err != nil { + logging.Warn(ctx, "git-refs push: rotate cut chunk to queue back failed", + slog.String("error", err.Error())) + } +} + +// adaptChunkSize updates the remembered chunk size after a batch: halved when +// the budget cut a chunk in flight (a link too slow for that many refs per +// budget would otherwise never land the head of the queue), doubled back toward +// checkpointRefPushChunkSize after a batch that landed everything. +func adaptChunkSize(ctx context.Context, queue *checkpoint.PushQueue, size int, batch chunkPushResult, budgetCut bool) { + next := size + switch { + case budgetCut && len(batch.failed) > 0: + next = max(1, size/2) + case len(batch.failed) == 0 && len(batch.untried) == 0: + next = min(checkpointRefPushChunkSize, size*2) + } + if next == size { + return + } + if err := queue.SetChunkSizeHint(next); err != nil { + logging.Debug(ctx, "git-refs push: remember chunk size failed", slog.String("error", err.Error())) + } +} + // refRetryResult is what the per-ref fallback of a flush left behind. type refRetryResult struct { pushed []plumbing.ReferenceName diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 58bc50754e..f9aab6a696 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -83,7 +83,7 @@ type chunkPushResult struct { unreachable string } -// pushRefChunks batch-pushes refs a chunk at a time (see +// pushRefChunks batch-pushes refs size at a time (see // checkpointRefPushChunkSize), calling onLanded with each chunk that lands so // the caller can dequeue it before a later chunk is cut. A failed chunk does not // stop the phase — a rejection fails only its own chunk — but an SSH auth @@ -91,10 +91,10 @@ type chunkPushResult struct { // row does, and so does an unreachable remote before any chunk landed: per-ref // retries cannot reach it either, and each would wait out its own connect // timeout. The first chunk is always attempted, matching the per-ref fallback. -func pushRefChunks(ctx context.Context, target string, refs []plumbing.ReferenceName, onLanded func([]plumbing.ReferenceName)) chunkPushResult { +func pushRefChunks(ctx context.Context, target string, refs []plumbing.ReferenceName, size int, onLanded func([]plumbing.ReferenceName)) chunkPushResult { var res chunkPushResult consecutive := 0 - for start := 0; start < len(refs); start += checkpointRefPushChunkSize { + for start := 0; start < len(refs); start += size { if start > 0 { if reason := chunkStopReason(ctx, consecutive); reason != "" { res.untried = refs[start:] @@ -104,7 +104,7 @@ func pushRefChunks(ctx context.Context, target string, refs []plumbing.Reference return res } } - chunk := refs[start:min(start+checkpointRefPushChunkSize, len(refs))] + chunk := refs[start:min(start+size, len(refs))] err := batchPushRefs(ctx, target, chunk) if err == nil { consecutive = 0 @@ -290,8 +290,9 @@ var checkpointPushBudget = 2 * time.Minute // checkpointPushBudget cannot: that one caps a single ref, and the fallback walks // the queue serially, so a backlog multiplies it. The batch needs the bound too: // it carries the whole backlog, so a slow uplink or a stalled connection would -// otherwise hold the user's git push for as long as the upload takes. A healthy push costs roughly one SSH round-trip per ref, so a -// few hundred queued refs is already tens of minutes of a `git push` that looks +// otherwise hold the user's git push for as long as the upload takes. A +// healthy push costs roughly one SSH round-trip per ref, so a few hundred +// queued refs is already tens of minutes of a `git push` that looks // hung; when the destination is unreachable every ref instead pays the full // per-ref budget and the same queue runs for hours. Refs that do not fit stay // queued and go out on the next push, so this bounds progress, never data. @@ -309,7 +310,11 @@ var checkpointPushBudget = 2 * time.Minute // Declared as a var so tests can shrink it. var checkpointFlushBudget = 2 * time.Minute -// checkpointRefPushChunkSize is how many queued refs one batch push carries. +// checkpointRefPushChunkSize is the most queued refs one batch push carries. +// A link too slow to finish that many within the flush budget would otherwise +// never land the chunk at the head of the queue, on any push, so the size +// actually used adapts (see PushQueue.ChunkSizeHint): a chunk cut by the budget +// halves it, down to one ref, and a flush that lands everything doubles it back. // Chunks are what make the flush budget bound progress rather than discard it: // each chunk that lands leaves the queue at once, so a backlog too large for one // budget drains over several pushes instead of being cut at the same point by diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index f122f2db64..067766a541 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -138,11 +138,12 @@ func TestFlushCheckpointRefs_StopsWhenBudgetExhausted(t *testing.T) { "nothing landed, so the whole queue stays") assert.LessOrEqual(t, countedAttempts(t, countFile), 2, "the deadline must cut the fallback, not let it walk the queue") - // An exhausted budget still buys one attempt: aborting before any work would - // starve the queue instead of draining it a little at a time. + // The batch always attempts its first chunk; a budget that cuts it ends + // the flush there rather than "retrying" on the spent budget. logged, readErr := os.ReadFile(filepath.Join(workDir, logging.LogsDir, "entire.log")) require.NoError(t, readErr) - assert.Contains(t, string(logged), `"attempted":1`, "a flush must always try at least one ref") + assert.Contains(t, string(logged), "batch push cut by the flush budget") + assert.NotContains(t, output, "retrying") remaining, err := queue.Drain() require.NoError(t, err) @@ -247,6 +248,7 @@ func refHashOf(t *testing.T, repo *git.Repository, ref plumbing.ReferenceName) s // of the queue starves exactly as it does behind a failing prefix — rotation is // fair scheduling, not a verdict on the ref that happened to be in flight. func TestFlushCheckpointRefs_BudgetAbortRotatesToo(t *testing.T) { + shrinkRefPushChunkSize(t, 1) // the cut chunk is exactly the first ref workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 4) prepareGitRefsPrePush(t, workDir, bareDir) installCheckpointRejectHook(t, bareDir, false, "") @@ -486,3 +488,74 @@ func TestFlushCheckpointRefs_UnreachableRemoteStopsAtOnce(t *testing.T) { require.NoError(t, err) assert.Equal(t, refs, remaining, "nothing is dropped or reordered") } + +// installSlowMultiRefHook stalls any push carrying more than one ref, and +// accepts single-ref pushes at once: a link too slow for a whole chunk. +func installSlowMultiRefHook(t *testing.T, bareDir string) { + t.Helper() + hook := "#!/bin/sh\nn=$(grep -c refs/entire/)\n[ \"$n\" -gt 1 ] && sleep 30\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) +} + +// TestFlushCheckpointRefs_ChunkTooSlowForBudgetShrinks pins progress on a +// link too slow to land one chunk within the budget: without adapting, the +// same head-of-queue chunk is cut on every push and nothing ever lands. +func TestFlushCheckpointRefs_ChunkTooSlowForBudgetShrinks(t *testing.T) { + shrinkRefPushChunkSize(t, 4) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 4) + prepareGitRefsPrePush(t, workDir, bareDir) + installSlowMultiRefHook(t, bareDir) + restoreBudget := checkpointFlushBudget + checkpointFlushBudget = 2 * time.Second + t.Cleanup(func() { checkpointFlushBudget = restoreBudget }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + assert.Contains(t, output, "Stopped pushing: budget (2s) exhausted; 4 checkpoint ref(s) stay queued") + assert.NotContains(t, output, "retrying", "a budget cut is not a push failure to retry") + assert.Equal(t, 2, queue.ChunkSizeHint(checkpointRefPushChunkSize), "the cut chunk halves the next one") + + for range 2 { // 2 per push is still too slow; then single refs land. + restore = captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + restore() + } + for _, ref := range refs { + assert.Equal(t, refHashOf(t, repo, ref), remoteRefHash(t, bareDir, ref)) + } + assert.Equal(t, 2, queue.ChunkSizeHint(checkpointRefPushChunkSize), "a clean flush grows it back") +} + +// TestPushQueuedCheckpointRefs_FallbackGetsFreshBudget: the migration push +// leaves its batch unbounded, so a slow batch must not leave its per-ref +// fallback a budget the batch already spent. +func TestPushQueuedCheckpointRefs_FallbackGetsFreshBudget(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 6) + prepareGitRefsPrePush(t, workDir, bareDir) + hook := "#!/bin/sh\nblocked=0\nwhile read -r old new ref; do\n" + + " [ \"$ref\" = '" + refs[0].String() + "' ] && blocked=1\ndone\nsleep 1.5\n" + + "if [ \"$blocked\" = 1 ]; then echo '" + checkpointRejectReason + "' >&2; exit 1; fi\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) + restoreBudget := checkpointFlushBudget + checkpointFlushBudget = 4 * time.Second + t.Cleanup(func() { checkpointFlushBudget = restoreBudget }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + _, err = flushCheckpointRefsQueue(t.Context(), repo, pushSettings{remote: bareDir}, false) + restore() + require.Error(t, err, "the blocked ref still fails") + assert.Equal(t, refHashOf(t, repo, refs[1]), remoteRefHash(t, bareDir, refs[1]), + "the blocked ref's healthy chunk-mate lands through the fallback despite the slow batch") +} diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index 2186b83b7a..1d3547759a 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -118,7 +118,7 @@ without additional secret redaction of the remote's diagnostic output. Plain non SSH authentication failures retain their dedicated hint. Failures remain queued and **never fail the user's git push**. -The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch that spends the budget leaves the fallback none, so its one guaranteed attempt fails at once and the refs stay queued. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it. Under the pre-push hook, OpenSSH also gets `ConnectTimeout` (`withBatchModeSSH`), so an unreachable host fails within seconds instead of spending the budget on a kernel connect timeout. Other ssh clients get only what they accept, mirroring git's own variant detection (`GIT_SSH_VARIANT`, `ssh.variant`, then the program name): plink gets `-batch`, its equivalent of `BatchMode`, never `-o`; TortoisePlink and the `simple` variant get nothing; an unrecognized wrapper keeps `BatchMode` only. A `GIT_SSH` program path is shell-quoted when rewritten into `GIT_SSH_COMMAND`, so a path with spaces stays one word. +The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch the budget cuts — between chunks or mid-chunk — ends the flush there ("Stopped pushing: budget … exhausted"); the fallback is not run on a spent budget, and the cut chunk rotates to the back of the queue. The chunk size adapts to the link (`PushQueue.ChunkSizeHint`, `entire-checkpoint-push-chunk-size` in the git common dir): a chunk the budget cut halves it, down to one ref, and a batch that lands everything doubles it back toward `checkpointRefPushChunkSize`. Without that, a link too slow to land one full chunk within the budget would cut the same chunk on every push and never make progress. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it, and opens the fallback's budget only after the batch. Under the pre-push hook, OpenSSH also gets `ConnectTimeout=60` (`withBatchModeSSH`), so an unreachable host fails within a minute instead of spending the budget on a kernel connect timeout. As a command-line option it overrides a `ConnectTimeout` in `~/.ssh/config`; 60s leaves room for slow-starting proxies, whose banner wait it also bounds. Other ssh clients get only what they accept, mirroring git's own variant detection (`GIT_SSH_VARIANT`, `ssh.variant`, then the program name — `auto` defers to the name): plink gets `-batch`, its equivalent of `BatchMode`, never `-o`; TortoisePlink and the `simple` variant get nothing; an unrecognized wrapper keeps `BatchMode` only. A `GIT_SSH` program path is shell-quoted when rewritten into `GIT_SSH_COMMAND`, so a path with spaces stays one word. When the git config that names the ssh command cannot be read, the environment is left unchanged rather than replacing the user's command with a guess. A remote that cannot be reached at all ends the flush after one attempt. When nothing has landed and a chunk's git output carries a connect-phase failure from ssh or curl (`remote.UnreachableRemoteLine`: unresolvable host, failed connect), later chunks and the per-ref fallback would each fail the same way after their own connect timeout, so the flush stops, prints that one line — credentials redacted, control bytes stripped, capped — and leaves the whole queue in order. The match is deliberately narrow: no bare strerror text (curl's is localized), no mid-transfer timeouts (the host was reachable), and no `remote:` lines (a reachable remote's own hook failing to resolve a host). A miss only costs the old path, the bounded fallback. diff --git a/docs/development/checkpoint-implementation.md b/docs/development/checkpoint-implementation.md index 7f301be26c..a6dec948b0 100644 --- a/docs/development/checkpoint-implementation.md +++ b/docs/development/checkpoint-implementation.md @@ -67,7 +67,7 @@ The manual-commit strategy (`manual_commit*.go`) does not modify the active bran - **Orphaned session state cleanup** (`isOrphanedSessionState`) - `listAllSessionStates` deletes a state only when it is ended (`State.IsEnded`: ENDED phase or `EndedAt` stamped), never condensed (no `LastCheckpointID`), and has no pending work (`!HasPendingWork()`). Legacy empty-phase records normalize to IDLE when listed, so they are no longer deleted here (they previously were) and instead remain until `StaleSessionThreshold` expiry. An IDLE never-condensed state is either a live session between turns — a read-only turn, or the turn right after a linked commit — or belongs to the exited-owner finalizer when `OwnerExited` is true. The store is listed by every worktree's hooks, so deleting either shape re-initialised live sessions from zero or erased the ownership evidence before `finalizeExitedSessions` could end the session cleanly. IDLE states therefore remain for lifecycle finalization or `StaleSessionThreshold` expiry. - PrePush hook can push `entire/checkpoints/v1` branch alongside user pushes - **Git-refs push failures stay fail-soft**: confirmed remote policy/hook rejections skip fetch/replay so repeated blocked pushes do not rewrite local commits. Other failures still attempt recovery; failed fetch/replay preserves the original push cause, not a presumed divergence. Confirmed remote rejections print one bounded example per flush with Git's line breaks intact (log errors remain single-line), without additional secret redaction of the remote's diagnostic output; unsuccessful refs stay queued. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). -- **The git-refs pre-push flush is bounded end to end**: the batch pushes `checkpointRefPushChunkSize` refs per round-trip, dequeuing each chunk as it lands, and in the pre-push path shares one `checkpointFlushBudget` deadline with the per-ref fallback (the explicit migration push leaves the batch unbounded). The batch stops after `maxConsecutiveChunkPushFailures` failed chunks in a row, or after the first chunk that fails to connect at all (`remote.UnreachableRemoteLine`; skips the fallback and prints that one redacted line), and the fallback after `maxConsecutiveRefPushFailures` failed refs — each always after at least one attempt, with skipped refs left queued for the next push. Each failed chunk's error is logged before the fallback runs so a transport failure, which `checkpointRefRejectionReason` deliberately does not match, still names its cause; and `remote.Fetch` annotates its error with git's redacted output via `errWithGitOutput` — deliberately not a `PushError`, which is how a refused push is recognised — rather than the caller substituting that output for the error. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). +- **The git-refs pre-push flush is bounded end to end**: the batch pushes `checkpointRefPushChunkSize` refs per round-trip, dequeuing each chunk as it lands, and in the pre-push path shares one `checkpointFlushBudget` deadline with the per-ref fallback (the explicit migration push leaves the batch unbounded). The chunk size adapts to the link (halved when the budget cuts a chunk, doubled back after a clean batch), and a budget cut ends the flush without running the fallback on the spent budget. The batch stops after `maxConsecutiveChunkPushFailures` failed chunks in a row, or after the first chunk that fails to connect at all (`remote.UnreachableRemoteLine`; skips the fallback and prints that one redacted line), and the fallback after `maxConsecutiveRefPushFailures` failed refs — each always after at least one attempt, with skipped refs left queued for the next push. Each failed chunk's error is logged before the fallback runs so a transport failure, which `checkpointRefRejectionReason` deliberately does not match, still names its cause; and `remote.Fetch` annotates its error with git's redacted output via `errWithGitOutput` — deliberately not a `PushError`, which is how a refused push is recognised — rather than the caller substituting that output for the error. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). - **Single checkpoint sync remote** - checkpoint data syncs only to the elected checkpoint sync remote (`strategy_options.checkpoint_push_remote` setting → captured election → `origin` → sole remote → first remote in `.git/config` order; fail-closed only when the setting names a missing remote — the captured tier is fail-soft), on both the git-branch and git-refs backends. **Capture**: during pre-push, a push whose target agrees with the branch's declared push destination (`branch..pushRemote` → `remote.pushDefault` → `branch..remote`) and differs from the current election persists that remote as the captured election (git common dir, `entire-checkpoint-sync-remotes.json`), announces it on stderr, and the same push carries the checkpoints — see `strategy/checkpoint_sync_capture.go`. Declaration alone is deliberately **not** a tier: electing from tracking config at rest silently drops checkpoint sync on every push to a different remote (`git push HEAD`, a `git clone -o base` pushing checkpoints to a separately added `origin`, any repo with `remote.pushDefault` — the `74e239a9` regression), and a bare push to an undeclared remote never elects (the pre-single-remote transcript leak). Phase-1 capture rules: at most one captured remote, first still-configured capture sticks (no per-push ping-pong in mixed-habit repos; a captured remote that was renamed or removed no longer vetoes the next capture), explicit `checkpoint_push_remote` outranks and disables capture. Pushes to any other remote or to a raw URL never carry checkpoint data; the dedicated `checkpoint_remote` URL mode is exempt. `entire status` shows the sync destination, its source (`observed` renders as "follows your branch's push destination"), and an unpushed-checkpoint count. - **A refused `checkpoint_remote` says so where it will be seen** - `checkpointRemoteIsInherited` refuses a committed `checkpoint_remote` whose owner does not match every remote identifying this repo, and used to say so only in a WARN. The visible symptom — checkpoints arriving in the code repository — is a working setup, just not the one that was asked for, so the rejection now reaches the user in `entire status` (text and `--json`, with `checkpoint_remote_ignored_verdict`) and `entire enable`, which take one verdict from `ignoredCheckpointRemote` and print the same sentence — "Checkpoints sync to X, not to the configured checkpoint_remote Y: reason" — plus the claim command built by `remote.ClaimCheckpointRemoteCommand`, for a disproved owner (a fork; rendered as information) and an unprovable one (a warning) alike. With `push_sessions` disabled both commands judge the read path instead, and also report a store the fetch side declined for a non-ownership reason. Pre-push stderr (`warnIgnoredCheckpointRemote`) is narrower: it warns only for an **unprovable** owner, because a disproved one is the ordinary fork contributor, for whom checkpoints landing in their fork is the correct outcome and a warning on every push would be noise they could silence only by claiming someone else's store. The repo is printed through `tuiutil.SanitizeDisplayText`, because it comes from the committed settings file. A local settings file present in the index or HEAD is never accepted as consent: `entire enable --local --checkpoint-remote` refuses to write it and says to untrack it, and the enable claim prompt is not offered. The enable claim prompt respects `--yes` and `--agent`, and any successful claim precedes destination resolution for the final report. It blocks nothing: checkpoints keep flowing to the elected remote, because refusing to push them turns a misconfiguration into lost work. The pre-push warning fires on **every push that delivered checkpoints** while the condition holds — a one-shot notice is seen by whoever set the repo up and not by whoever hits the problem — and is emitted after delivery, so a push carrying no checkpoints says nothing. Two gates guard it and neither is redundant: `checkpointRemoteConfigured` is the free half (an empty `checkpointURL` means "none configured" as often as "configured and refused"), and `hasCheckpointURL` is authoritative, because `PushURL` decided where this push goes while `InheritedCheckpointRemote` re-runs the vote over an identity set that can diverge from it. The claim command is offered **only for providers `parseCheckpointRemoteFlag` accepts** (github, and gitlab including nested groups): `GetCheckpointRemote` validates no provider at all, so naming any other would print a command that fails when pasted. - **Checkpoint reads follow the election** - reads consult an ordered candidate chain: the elected sync remote first, then `origin` as a **read-only legacy tier** (pre-single-remote checkpoints live there; `strategy.CheckpointReadRemotes` / `CheckpointReadRemotesWithElection` resolve the chain, failing OPEN to `[origin]` on an election error). Metadata-branch fetches refresh every candidate tracking ref because branch existence does not prove a requested checkpoint exists there; other fetch targets, tracking-ref readers, and blob hydration try candidates in order. Git-refs remote discovery ls-remotes every candidate **without** needing a dedicated `checkpoint_remote` and merges the listings. Local-ref writers stay **elected-remote-only**: `EnsurePrimaryRef`, the metadata-fetch advance, and `promoteRemoteTrackingPrimary` never act on the legacy tier (a stale origin feeding `SafelyAdvanceLocalRef` is the #1374-class hazard), keyed on the explicit election result, never on the chain's first entry. A remoteless repo keeps its "checkpoint absent" classification, which requires positive evidence (successful empty `git remote` listing, live ctx, readable settings without a `checkpoint_remote` key). From 62cdaf52743dbc81c51a3717d36e322ecd48d2d6 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 18:47:32 -0700 Subject: [PATCH 04/16] fix(checkpoint): act on landed checkpoint refs even when a later chunk fails A chunked flush can land chunks and then fail a later one (an SSH auth failure, say). The pre-push read any error as "nothing synced" and skipped committing the captured sync remote and the ignored-checkpoint_remote warning, though the landed checkpoints reached the remote. Act on the landed count instead. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4F5G5ZMZ7TWB7KMA9A33T6D --- cmd/entire/cli/strategy/manual_commit_push.go | 26 ++++++++++--------- .../cli/strategy/refs_push_bounds_test.go | 25 ++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index be70d6b2c5..95dc83689e 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -421,23 +421,25 @@ func (s *ManualCommitStrategy) prePushCheckpointRefs(ctx context.Context, ps pus return nil } - if flushed, err := flushCheckpointRefsQueue(ctx, repo, ps, true); err == nil { - // Delivered, and only if something actually was: an empty queue pushed - // nothing, so it must not move the election or announce that it had. - if pendingCapture != "" && flushed > 0 { - commitCapturedSyncRemote(ctx, pendingCapture) - } - // An empty queue carried no checkpoints, so there is nothing to warn - // was misdirected — see warnIgnoredCheckpointRemote. - if flushed > 0 { - warnIgnoredCheckpointRemote(ctx, ps) - } - } else { + flushed, err := flushCheckpointRefsQueue(ctx, repo, ps, true) + if err != nil { // Fail-soft: a checkpoint-ref push failure must never block the user's // git push. The refs stay queued for the next pre-push. logging.Warn(ctx, "git-refs pre-push: checkpoint ref push failed; refs left queued", slog.String("error", err.Error())) } + // Delivered, and only if something actually was — counted, not inferred + // from the error: a chunked flush can land chunks and still fail later + // ones, and those checkpoints reached the remote all the same. An empty + // queue pushed nothing, so it must not move the election or announce that + // it had, nor warn that checkpoints were misdirected (see + // warnIgnoredCheckpointRemote). + if flushed > 0 { + if pendingCapture != "" { + commitCapturedSyncRemote(ctx, pendingCapture) + } + warnIgnoredCheckpointRemote(ctx, ps) + } return nil } diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index 067766a541..942ec2ab31 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" + checkpointremote "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" "github.com/entireio/cli/cmd/entire/cli/entiredir" "github.com/entireio/cli/cmd/entire/cli/gitrepo" "github.com/entireio/cli/cmd/entire/cli/logging" @@ -559,3 +560,27 @@ func TestPushQueuedCheckpointRefs_FallbackGetsFreshBudget(t *testing.T) { assert.Equal(t, refHashOf(t, repo, refs[1]), remoteRefHash(t, bareDir, refs[1]), "the blocked ref's healthy chunk-mate lands through the fallback despite the slow batch") } + +// TestFlushCheckpointRefs_PartialDeliveryStillCountsAsDelivered: a chunked +// flush can land chunks and then fail (here, SSH auth on a later chunk). The +// landed chunks reached the remote, so the pre-push acts on them — capture, +// misdirection warning — rather than reading the error as "nothing synced". +func TestFlushCheckpointRefs_PartialDeliveryStillCountsAsDelivered(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 4) + prepareGitRefsPrePush(t, workDir, bareDir) + hook := "#!/bin/sh\ngrep -q '" + refs[2].String() + "' && { echo 'git@github.com: Permission denied (publickey).' >&2; exit 1; }\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + pushed, err := flushCheckpointRefsQueue(checkpointremote.WithNonInteractiveSSH(t.Context()), repo, pushSettings{remote: bareDir}, true) + output := restore() + require.Error(t, err) + assert.Equal(t, 2, pushed, "the first chunk landed before the auth failure and is reported as pushed") + assert.Contains(t, output, "SSH authentication failed") +} From 35c2323c6d615c0e40cccca1583121aba432a624 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 19:04:22 -0700 Subject: [PATCH 05/16] fix(checkpoint): a rejected chunk proves the remote reachable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unreachable-remote stop fired whenever nothing had landed yet, even after an earlier chunk was rejected — which equally proves the remote answered. A later transient connect error then abandoned the flush and skipped the per-ref fallback that could still land or recover those refs. Only a connect failure on the first chunk the remote saw now counts. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4F6F0VGSQ9J2E6Q47W4G180 --- cmd/entire/cli/strategy/push_common.go | 8 +++-- .../cli/strategy/refs_push_bounds_test.go | 35 +++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index f9aab6a696..4cc2eed464 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -130,9 +130,11 @@ func pushRefChunks(ctx context.Context, target string, refs []plumbing.Reference res.untried = refs[start+len(chunk):] return res } - // A remote that already took a chunk this flush was reachable; a later - // connect failure is transient, and the per-ref fallback may still land. - if line, ok := remote.UnreachableRemoteLine(err); ok && res.landed == 0 { + // A remote that already answered this flush — took a chunk, or + // rejected one — was reachable; a later connect failure is transient, + // and the per-ref fallback may still land or recover those refs. Only a + // connect failure on the first chunk the remote saw means unreachable. + if line, ok := remote.UnreachableRemoteLine(err); ok && res.landed == 0 && len(res.failed) == len(chunk) { res.unreachable = line res.untried = refs[start+len(chunk):] return res diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index 942ec2ab31..47656468dd 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -584,3 +584,38 @@ func TestFlushCheckpointRefs_PartialDeliveryStillCountsAsDelivered(t *testing.T) assert.Equal(t, 2, pushed, "the first chunk landed before the auth failure and is reported as pushed") assert.Contains(t, output, "SSH authentication failed") } + +// TestFlushCheckpointRefs_RejectionProvesRemoteReachable: once the remote has +// answered a chunk — even with a rejection — a later connect error is +// transient, not "unreachable": the flush carries on to the per-ref fallback +// instead of abandoning refs it could still land. +func TestFlushCheckpointRefs_RejectionProvesRemoteReachable(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 4) + prepareGitRefsPrePush(t, workDir, "ssh://git@example.invalid"+bareDir) + installBlockRefHook(t, bareDir, refs[0].String()) + + // A fake ssh that runs the remote command locally, except the second + // connection, which fails the way an unreachable host does. + countFile := filepath.Join(t.TempDir(), "attempts") + fakeSSH := filepath.Join(t.TempDir(), "ssh") + script := "#!/bin/sh\necho attempt >> '" + countFile + "'\n" + + "n=$(wc -l < '" + countFile + "' | tr -d ' ')\n" + + "if [ \"$n\" = 2 ]; then echo 'ssh: connect to host example.invalid port 22: Connection refused' >&2; exit 255; fi\n" + + "for last; do :; done\nexec sh -c \"$last\"\n" + require.NoError(t, os.WriteFile(fakeSSH, []byte(script), 0o755)) + t.Setenv("GIT_SSH_COMMAND", fakeSSH) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + assert.NotContains(t, output, "Couldn't reach", "the remote answered the first chunk") + assert.Equal(t, refHashOf(t, repo, refs[1]), remoteRefHash(t, bareDir, refs[1]), + "the rejected chunk's healthy ref lands through the fallback") +} From 8124d4debb509b3cb4ac799bf80fa7280fe406b1 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 19:12:19 -0700 Subject: [PATCH 06/16] fix(checkpoint): push checkpoint refs in chunks of up to 200 Measured against GitHub (ssh) and Entire, 200 checkpoints (~20MB) went out in 7s / 4s as one push but took 20s / 11s in chunks of 25: every push pays a fixed cost (the connection, and the remote advertising every checkpoint ref it holds), so small chunks made every healthy push about three times slower. Raise the ceiling to 200, which restores the single-push speed on healthy links, and quarter the adaptive size when the budget cuts a chunk so a slow link still converges in a few pushes. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4F6XJB2HCN41C4BPA9ZFF03 --- cmd/entire/cli/strategy/manual_commit_push.go | 7 ++++--- cmd/entire/cli/strategy/push_common.go | 15 ++++++++++----- cmd/entire/cli/strategy/refs_push_bounds_test.go | 4 ++-- docs/architecture/ref-checkpoint-backend.md | 2 +- docs/development/checkpoint-implementation.md | 2 +- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index 95dc83689e..135101879c 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -705,15 +705,16 @@ func stopOnBudgetCut(ctx, flushCtx context.Context, queue *checkpoint.PushQueue, } } -// adaptChunkSize updates the remembered chunk size after a batch: halved when -// the budget cut a chunk in flight (a link too slow for that many refs per +// adaptChunkSize updates the remembered chunk size after a batch: quartered +// when the budget cut a chunk in flight — steeply, since each cut costs a whole +// budget with nothing landed — (a link too slow for that many refs per // budget would otherwise never land the head of the queue), doubled back toward // checkpointRefPushChunkSize after a batch that landed everything. func adaptChunkSize(ctx context.Context, queue *checkpoint.PushQueue, size int, batch chunkPushResult, budgetCut bool) { next := size switch { case budgetCut && len(batch.failed) > 0: - next = max(1, size/2) + next = max(1, size/4) case len(batch.failed) == 0 && len(batch.untried) == 0: next = min(checkpointRefPushChunkSize, size*2) } diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 4cc2eed464..262947c30c 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -313,15 +313,20 @@ var checkpointPushBudget = 2 * time.Minute var checkpointFlushBudget = 2 * time.Minute // checkpointRefPushChunkSize is the most queued refs one batch push carries. -// A link too slow to finish that many within the flush budget would otherwise -// never land the chunk at the head of the queue, on any push, so the size -// actually used adapts (see PushQueue.ChunkSizeHint): a chunk cut by the budget -// halves it, down to one ref, and a flush that lands everything doubles it back. +// Large on purpose: every push pays a fixed cost — the connection and the +// remote advertising every ref it holds, one per checkpoint — so a healthy +// link is fastest with the fewest pushes. Against GitHub and Entire, 200 +// checkpoints (~20MB) went out in 3-7s as one push, and took 11-20s in chunks +// of 25. A link too slow to finish a chunk within the flush budget would +// otherwise never land the chunk at the head of the queue, on any push, so the +// size actually used adapts (see PushQueue.ChunkSizeHint): a chunk cut by the +// budget quarters it, down to one ref, and a flush that lands everything +// doubles it back. // Chunks are what make the flush budget bound progress rather than discard it: // each chunk that lands leaves the queue at once, so a backlog too large for one // budget drains over several pushes instead of being cut at the same point by // every one of them. Declared as a var so tests can shrink it. -var checkpointRefPushChunkSize = 25 +var checkpointRefPushChunkSize = 200 // maxConsecutiveChunkPushFailures stops the batch phase once this many chunks in // a row have failed. A rejection fails only its own chunk, so batching carries on diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index 47656468dd..92200b76d3 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -520,9 +520,9 @@ func TestFlushCheckpointRefs_ChunkTooSlowForBudgetShrinks(t *testing.T) { output := restore() assert.Contains(t, output, "Stopped pushing: budget (2s) exhausted; 4 checkpoint ref(s) stay queued") assert.NotContains(t, output, "retrying", "a budget cut is not a push failure to retry") - assert.Equal(t, 2, queue.ChunkSizeHint(checkpointRefPushChunkSize), "the cut chunk halves the next one") + assert.Equal(t, 1, queue.ChunkSizeHint(checkpointRefPushChunkSize), "the cut chunk quarters the next one") - for range 2 { // 2 per push is still too slow; then single refs land. + for range 2 { // single refs land; the next push finds nothing queued. restore = captureStderr(t) require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) restore() diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index 1d3547759a..491d3641a7 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -118,7 +118,7 @@ without additional secret redaction of the remote's diagnostic output. Plain non SSH authentication failures retain their dedicated hint. Failures remain queued and **never fail the user's git push**. -The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch the budget cuts — between chunks or mid-chunk — ends the flush there ("Stopped pushing: budget … exhausted"); the fallback is not run on a spent budget, and the cut chunk rotates to the back of the queue. The chunk size adapts to the link (`PushQueue.ChunkSizeHint`, `entire-checkpoint-push-chunk-size` in the git common dir): a chunk the budget cut halves it, down to one ref, and a batch that lands everything doubles it back toward `checkpointRefPushChunkSize`. Without that, a link too slow to land one full chunk within the budget would cut the same chunk on every push and never make progress. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it, and opens the fallback's budget only after the batch. Under the pre-push hook, OpenSSH also gets `ConnectTimeout=60` (`withBatchModeSSH`), so an unreachable host fails within a minute instead of spending the budget on a kernel connect timeout. As a command-line option it overrides a `ConnectTimeout` in `~/.ssh/config`; 60s leaves room for slow-starting proxies, whose banner wait it also bounds. Other ssh clients get only what they accept, mirroring git's own variant detection (`GIT_SSH_VARIANT`, `ssh.variant`, then the program name — `auto` defers to the name): plink gets `-batch`, its equivalent of `BatchMode`, never `-o`; TortoisePlink and the `simple` variant get nothing; an unrecognized wrapper keeps `BatchMode` only. A `GIT_SSH` program path is shell-quoted when rewritten into `GIT_SSH_COMMAND`, so a path with spaces stays one word. When the git config that names the ssh command cannot be read, the environment is left unchanged rather than replacing the user's command with a guess. +The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch the budget cuts — between chunks or mid-chunk — ends the flush there ("Stopped pushing: budget … exhausted"); the fallback is not run on a spent budget, and the cut chunk rotates to the back of the queue. The chunk size adapts to the link (`PushQueue.ChunkSizeHint`, `entire-checkpoint-push-chunk-size` in the git common dir): a chunk the budget cut quarters it, down to one ref, and a batch that lands everything doubles it back toward `checkpointRefPushChunkSize` (200). The ceiling is high on purpose: every push pays a fixed cost — the connection plus the remote advertising every checkpoint ref it holds — so a healthy link is fastest in as few pushes as possible. Measured against GitHub and Entire, 200 checkpoints (~20MB) took 3–7s as one push and 11–20s in chunks of 25. Without that, a link too slow to land one full chunk within the budget would cut the same chunk on every push and never make progress. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it, and opens the fallback's budget only after the batch. Under the pre-push hook, OpenSSH also gets `ConnectTimeout=60` (`withBatchModeSSH`), so an unreachable host fails within a minute instead of spending the budget on a kernel connect timeout. As a command-line option it overrides a `ConnectTimeout` in `~/.ssh/config`; 60s leaves room for slow-starting proxies, whose banner wait it also bounds. Other ssh clients get only what they accept, mirroring git's own variant detection (`GIT_SSH_VARIANT`, `ssh.variant`, then the program name — `auto` defers to the name): plink gets `-batch`, its equivalent of `BatchMode`, never `-o`; TortoisePlink and the `simple` variant get nothing; an unrecognized wrapper keeps `BatchMode` only. A `GIT_SSH` program path is shell-quoted when rewritten into `GIT_SSH_COMMAND`, so a path with spaces stays one word. When the git config that names the ssh command cannot be read, the environment is left unchanged rather than replacing the user's command with a guess. A remote that cannot be reached at all ends the flush after one attempt. When nothing has landed and a chunk's git output carries a connect-phase failure from ssh or curl (`remote.UnreachableRemoteLine`: unresolvable host, failed connect), later chunks and the per-ref fallback would each fail the same way after their own connect timeout, so the flush stops, prints that one line — credentials redacted, control bytes stripped, capped — and leaves the whole queue in order. The match is deliberately narrow: no bare strerror text (curl's is localized), no mid-transfer timeouts (the host was reachable), and no `remote:` lines (a reachable remote's own hook failing to resolve a host). A miss only costs the old path, the bounded fallback. diff --git a/docs/development/checkpoint-implementation.md b/docs/development/checkpoint-implementation.md index a6dec948b0..66af2ef7dd 100644 --- a/docs/development/checkpoint-implementation.md +++ b/docs/development/checkpoint-implementation.md @@ -67,7 +67,7 @@ The manual-commit strategy (`manual_commit*.go`) does not modify the active bran - **Orphaned session state cleanup** (`isOrphanedSessionState`) - `listAllSessionStates` deletes a state only when it is ended (`State.IsEnded`: ENDED phase or `EndedAt` stamped), never condensed (no `LastCheckpointID`), and has no pending work (`!HasPendingWork()`). Legacy empty-phase records normalize to IDLE when listed, so they are no longer deleted here (they previously were) and instead remain until `StaleSessionThreshold` expiry. An IDLE never-condensed state is either a live session between turns — a read-only turn, or the turn right after a linked commit — or belongs to the exited-owner finalizer when `OwnerExited` is true. The store is listed by every worktree's hooks, so deleting either shape re-initialised live sessions from zero or erased the ownership evidence before `finalizeExitedSessions` could end the session cleanly. IDLE states therefore remain for lifecycle finalization or `StaleSessionThreshold` expiry. - PrePush hook can push `entire/checkpoints/v1` branch alongside user pushes - **Git-refs push failures stay fail-soft**: confirmed remote policy/hook rejections skip fetch/replay so repeated blocked pushes do not rewrite local commits. Other failures still attempt recovery; failed fetch/replay preserves the original push cause, not a presumed divergence. Confirmed remote rejections print one bounded example per flush with Git's line breaks intact (log errors remain single-line), without additional secret redaction of the remote's diagnostic output; unsuccessful refs stay queued. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). -- **The git-refs pre-push flush is bounded end to end**: the batch pushes `checkpointRefPushChunkSize` refs per round-trip, dequeuing each chunk as it lands, and in the pre-push path shares one `checkpointFlushBudget` deadline with the per-ref fallback (the explicit migration push leaves the batch unbounded). The chunk size adapts to the link (halved when the budget cuts a chunk, doubled back after a clean batch), and a budget cut ends the flush without running the fallback on the spent budget. The batch stops after `maxConsecutiveChunkPushFailures` failed chunks in a row, or after the first chunk that fails to connect at all (`remote.UnreachableRemoteLine`; skips the fallback and prints that one redacted line), and the fallback after `maxConsecutiveRefPushFailures` failed refs — each always after at least one attempt, with skipped refs left queued for the next push. Each failed chunk's error is logged before the fallback runs so a transport failure, which `checkpointRefRejectionReason` deliberately does not match, still names its cause; and `remote.Fetch` annotates its error with git's redacted output via `errWithGitOutput` — deliberately not a `PushError`, which is how a refused push is recognised — rather than the caller substituting that output for the error. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). +- **The git-refs pre-push flush is bounded end to end**: the batch pushes `checkpointRefPushChunkSize` refs per round-trip, dequeuing each chunk as it lands, and in the pre-push path shares one `checkpointFlushBudget` deadline with the per-ref fallback (the explicit migration push leaves the batch unbounded). The chunk size adapts to the link (up to 200 refs; quartered when the budget cuts a chunk, doubled back after a clean batch), and a budget cut ends the flush without running the fallback on the spent budget. The batch stops after `maxConsecutiveChunkPushFailures` failed chunks in a row, or after the first chunk that fails to connect at all (`remote.UnreachableRemoteLine`; skips the fallback and prints that one redacted line), and the fallback after `maxConsecutiveRefPushFailures` failed refs — each always after at least one attempt, with skipped refs left queued for the next push. Each failed chunk's error is logged before the fallback runs so a transport failure, which `checkpointRefRejectionReason` deliberately does not match, still names its cause; and `remote.Fetch` annotates its error with git's redacted output via `errWithGitOutput` — deliberately not a `PushError`, which is how a refused push is recognised — rather than the caller substituting that output for the error. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). - **Single checkpoint sync remote** - checkpoint data syncs only to the elected checkpoint sync remote (`strategy_options.checkpoint_push_remote` setting → captured election → `origin` → sole remote → first remote in `.git/config` order; fail-closed only when the setting names a missing remote — the captured tier is fail-soft), on both the git-branch and git-refs backends. **Capture**: during pre-push, a push whose target agrees with the branch's declared push destination (`branch..pushRemote` → `remote.pushDefault` → `branch..remote`) and differs from the current election persists that remote as the captured election (git common dir, `entire-checkpoint-sync-remotes.json`), announces it on stderr, and the same push carries the checkpoints — see `strategy/checkpoint_sync_capture.go`. Declaration alone is deliberately **not** a tier: electing from tracking config at rest silently drops checkpoint sync on every push to a different remote (`git push HEAD`, a `git clone -o base` pushing checkpoints to a separately added `origin`, any repo with `remote.pushDefault` — the `74e239a9` regression), and a bare push to an undeclared remote never elects (the pre-single-remote transcript leak). Phase-1 capture rules: at most one captured remote, first still-configured capture sticks (no per-push ping-pong in mixed-habit repos; a captured remote that was renamed or removed no longer vetoes the next capture), explicit `checkpoint_push_remote` outranks and disables capture. Pushes to any other remote or to a raw URL never carry checkpoint data; the dedicated `checkpoint_remote` URL mode is exempt. `entire status` shows the sync destination, its source (`observed` renders as "follows your branch's push destination"), and an unpushed-checkpoint count. - **A refused `checkpoint_remote` says so where it will be seen** - `checkpointRemoteIsInherited` refuses a committed `checkpoint_remote` whose owner does not match every remote identifying this repo, and used to say so only in a WARN. The visible symptom — checkpoints arriving in the code repository — is a working setup, just not the one that was asked for, so the rejection now reaches the user in `entire status` (text and `--json`, with `checkpoint_remote_ignored_verdict`) and `entire enable`, which take one verdict from `ignoredCheckpointRemote` and print the same sentence — "Checkpoints sync to X, not to the configured checkpoint_remote Y: reason" — plus the claim command built by `remote.ClaimCheckpointRemoteCommand`, for a disproved owner (a fork; rendered as information) and an unprovable one (a warning) alike. With `push_sessions` disabled both commands judge the read path instead, and also report a store the fetch side declined for a non-ownership reason. Pre-push stderr (`warnIgnoredCheckpointRemote`) is narrower: it warns only for an **unprovable** owner, because a disproved one is the ordinary fork contributor, for whom checkpoints landing in their fork is the correct outcome and a warning on every push would be noise they could silence only by claiming someone else's store. The repo is printed through `tuiutil.SanitizeDisplayText`, because it comes from the committed settings file. A local settings file present in the index or HEAD is never accepted as consent: `entire enable --local --checkpoint-remote` refuses to write it and says to untrack it, and the enable claim prompt is not offered. The enable claim prompt respects `--yes` and `--agent`, and any successful claim precedes destination resolution for the final report. It blocks nothing: checkpoints keep flowing to the elected remote, because refusing to push them turns a misconfiguration into lost work. The pre-push warning fires on **every push that delivered checkpoints** while the condition holds — a one-shot notice is seen by whoever set the repo up and not by whoever hits the problem — and is emitted after delivery, so a push carrying no checkpoints says nothing. Two gates guard it and neither is redundant: `checkpointRemoteConfigured` is the free half (an empty `checkpointURL` means "none configured" as often as "configured and refused"), and `hasCheckpointURL` is authoritative, because `PushURL` decided where this push goes while `InheritedCheckpointRemote` re-runs the vote over an identity set that can diverge from it. The claim command is offered **only for providers `parseCheckpointRemoteFlag` accepts** (github, and gitlab including nested groups): `GetCheckpointRemote` validates no provider at all, so naming any other would print a command that fails when pasted. - **Checkpoint reads follow the election** - reads consult an ordered candidate chain: the elected sync remote first, then `origin` as a **read-only legacy tier** (pre-single-remote checkpoints live there; `strategy.CheckpointReadRemotes` / `CheckpointReadRemotesWithElection` resolve the chain, failing OPEN to `[origin]` on an election error). Metadata-branch fetches refresh every candidate tracking ref because branch existence does not prove a requested checkpoint exists there; other fetch targets, tracking-ref readers, and blob hydration try candidates in order. Git-refs remote discovery ls-remotes every candidate **without** needing a dedicated `checkpoint_remote` and merges the listings. Local-ref writers stay **elected-remote-only**: `EnsurePrimaryRef`, the metadata-fetch advance, and `promoteRemoteTrackingPrimary` never act on the legacy tier (a stale origin feeding `SafelyAdvanceLocalRef` is the #1374-class hazard), keyed on the explicit election result, never on the chain's first entry. A remoteless repo keeps its "checkpoint absent" classification, which requires positive evidence (successful empty `git remote` listing, live ctx, readable settings without a `checkpoint_remote` key). From 028f038e02ba1a7a273df5d7b4b5f6974685d6b8 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 20:12:14 -0700 Subject: [PATCH 07/16] fix(checkpoint): keep chunk size while draining; split rejected chunks Adversarial review fixes for the adaptive chunked flush: - The budget almost always cuts a long drain mid-chunk, so shrinking on any budget cut quartered the chunk size on every push of a backlog that was draining fine, down to one ref per push. Shrink only when nothing landed. - A forge declines a whole push for one bad ref (push protection, a ruleset), so with 200-ref chunks one secret sent 200 refs to the per-ref fallback and took several budgets to reach. Failed chunks are now re-pushed in halves, recursing only while the sibling half landed, which isolates a single bad ref in about 2*log2(chunk) pushes; the rest goes per-ref. - An end-to-end test now pins that a partial delivery captures the election (the earlier unit test passed without that fix). Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4FAB92AEF0JSB43RDPS5BZJ --- .../checkpoint_sync_remote_test.go | 44 +++++++++++ cmd/entire/cli/strategy/manual_commit_push.go | 28 +++++-- cmd/entire/cli/strategy/push_common.go | 67 ++++++++++++++++- .../cli/strategy/refs_push_bounds_test.go | 74 +++++++++++++++++-- docs/architecture/ref-checkpoint-backend.md | 4 +- 5 files changed, 200 insertions(+), 17 deletions(-) diff --git a/cmd/entire/cli/integration_test/checkpoint_sync_remote_test.go b/cmd/entire/cli/integration_test/checkpoint_sync_remote_test.go index 77d9a036b8..22f928e35b 100644 --- a/cmd/entire/cli/integration_test/checkpoint_sync_remote_test.go +++ b/cmd/entire/cli/integration_test/checkpoint_sync_remote_test.go @@ -859,3 +859,47 @@ func statusSyncJSONOutput(t *testing.T, env *TestEnv) statusSyncJSON { } return parsed } + +// TestCheckpointSyncRemote_PartialDeliveryCapturesElection: a chunked git-refs +// flush can land some checkpoints and fail others. The landed ones reached the +// declared destination, so the push captures the election even though the +// flush as a whole reported an error — reading any error as "nothing synced" +// would leave the election on a remote that received nothing. +func TestCheckpointSyncRemote_PartialDeliveryCapturesElection(t *testing.T) { + t.Parallel() + env := NewFeatureBranchEnv(t) + env.CheckpointStore = StoreGitRefs + env.SetupBareRemote() + bareFork := env.SetupNamedBareRemote(forkRemote) // `-u`: branch declares fork + + first := createCheckpointedCommit(t, env, "Add first module", "first.go", "package first", "Add first module") + second := createCheckpointedCommit(t, env, "Add second module", "second.go", "package second", "Add second module") + if first == "" || second == "" || first == second { + t.Fatalf("need two checkpoints, got %q and %q", first, second) + } + firstRef := "refs/entire/checkpoints/" + first[len(first)-2:] + "/" + first + + // One ref per push, so the first checkpoint lands on its own; the fork + // then refuses any push carrying another checkpoint ref. + if err := os.WriteFile(filepath.Join(env.RepoDir, ".git", "entire-checkpoint-push-chunk-size"), []byte("1\n"), 0o600); err != nil { + t.Fatal(err) + } + hook := "#!/bin/sh\nwhile read -r old new ref; do\n" + + " case \"$ref\" in refs/entire/checkpoints/*) [ \"$ref\" = '" + firstRef + "' ] || { echo 'declined' >&2; exit 1; } ;; esac\n" + + "done\nexit 0\n" + if err := os.WriteFile(filepath.Join(bareFork, "hooks", "pre-receive"), []byte(hook), 0o755); err != nil { + t.Fatal(err) + } + + env.RunPrePush(forkRemote) + + if !env.CheckpointExistsOnRemote(bareFork, first) { + t.Fatalf("precondition: checkpoint %s should have landed on fork", first) + } + if env.CheckpointExistsOnRemote(bareFork, second) { + t.Fatalf("precondition: checkpoint %s should have been refused", second) + } + if got := capturedSyncRemotesOnDisk(t, env); len(got) != 1 || got[0] != forkRemote { + t.Errorf("a push that landed checkpoints on the declared destination should capture it, got %v", got) + } +} diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index 135101879c..2fa1fac214 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -589,10 +589,20 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push // implies the remote answered — sends them after the wrong problem. fmt.Fprintf(os.Stderr, "[entire] Checkpoint ref push failed; retrying %d ref(s) individually...", len(batch.failed)) stop = startProgressDots(os.Stderr) - retry := retryRefsIndividually(ctx, flushCtx, dest.target, batch.failed, len(existing)) + // A forge that declines a push declines all of it (a ruleset, push + // protection), so one bad ref fails its whole chunk. Halving the chunk + // isolates it in a handful of pushes, where walking a 200-ref chunk one ref + // at a time would spend several budgets to reach it. + split := splitFailedChunks(ctx, flushCtx, dest.target, batch.failed, chunkSize, func(landed []plumbing.ReferenceName) { + if removeErr := queue.Remove(landed); removeErr != nil { + logging.Warn(ctx, "git-refs push: clear pushed refs from queue failed", + slog.String("error", removeErr.Error())) + } + }) + retry := retryRefsIndividually(ctx, flushCtx, dest.target, split.unresolved, len(existing)) pushed, failed, firstErr, abortReason := retry.pushed, retry.failed, retry.firstErr, retry.abortReason attempted := len(pushed) + len(failed) - totalPushed := batch.landed + len(pushed) + totalPushed := batch.landed + split.landed + len(pushed) stop(fmt.Sprintf(" pushed %d of %d", totalPushed, len(existing))) // The fallback reached every failed ref, but the batch stopped before // trying the rest of the queue: report that stop instead. @@ -705,15 +715,17 @@ func stopOnBudgetCut(ctx, flushCtx context.Context, queue *checkpoint.PushQueue, } } -// adaptChunkSize updates the remembered chunk size after a batch: quartered -// when the budget cut a chunk in flight — steeply, since each cut costs a whole -// budget with nothing landed — (a link too slow for that many refs per -// budget would otherwise never land the head of the queue), doubled back toward -// checkpointRefPushChunkSize after a batch that landed everything. +// adaptChunkSize updates the remembered chunk size after a batch. It is +// quartered when the budget cut a chunk in flight and nothing landed — a link +// too slow to land one chunk per budget would otherwise never move the head of +// the queue, and each such cut costs a whole budget, hence the steep step. A +// cut after chunks landed is a backlog draining as intended, not a chunk too +// large, so the size is kept. It doubles back toward checkpointRefPushChunkSize +// after a batch that landed everything. func adaptChunkSize(ctx context.Context, queue *checkpoint.PushQueue, size int, batch chunkPushResult, budgetCut bool) { next := size switch { - case budgetCut && len(batch.failed) > 0: + case budgetCut && batch.landed == 0 && len(batch.failed) > 0: next = max(1, size/4) case len(batch.failed) == 0 && len(batch.untried) == 0: next = min(checkpointRefPushChunkSize, size*2) diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 262947c30c..bbc3d8ce5b 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -130,8 +130,9 @@ func pushRefChunks(ctx context.Context, target string, refs []plumbing.Reference res.untried = refs[start+len(chunk):] return res } - // A remote that already answered this flush — took a chunk, or - // rejected one — was reachable; a later connect failure is transient, + // A remote that already answered this flush — took a chunk, or failed + // one any way other than a connect failure — was reachable; a later + // connect failure is transient, // and the per-ref fallback may still land or recover those refs. Only a // connect failure on the first chunk the remote saw means unreachable. if line, ok := remote.UnreachableRemoteLine(err); ok && res.landed == 0 && len(res.failed) == len(chunk) { @@ -143,6 +144,68 @@ func pushRefChunks(ctx context.Context, target string, refs []plumbing.Reference return res } +// splitFloor is the smallest failed batch splitFailedChunks halves; smaller +// ones go straight to the per-ref fallback, which retries each ref anyway. +const splitFloor = 4 + +// splitResult is what splitFailedChunks left for the per-ref fallback. +type splitResult struct { + landed int + unresolved []plumbing.ReferenceName +} + +// splitFailedChunks re-pushes each failed chunk of refs (chunkSize apart) in +// halves, recursing into a half only while its sibling landed: a single bad +// ref is then isolated in about 2·log2(chunk) pushes. When both halves fail +// the cause is not one ref (several diverged, or the destination), so both go +// to the per-ref fallback whole, as does everything once the budget is spent +// or the destination refuses the key or the connection. onLanded is called +// with each half that lands. +func splitFailedChunks(ctx, flushCtx context.Context, target string, refs []plumbing.ReferenceName, chunkSize int, + onLanded func([]plumbing.ReferenceName), +) splitResult { + var res splitResult + halt := false + var walk func([]plumbing.ReferenceName) + walk = func(batch []plumbing.ReferenceName) { + if halt || len(batch) <= splitFloor || flushCtx.Err() != nil { + res.unresolved = append(res.unresolved, batch...) + return + } + mid := len(batch) / 2 + var failedHalves [][]plumbing.ReferenceName + for _, half := range [][]plumbing.ReferenceName{batch[:mid], batch[mid:]} { + if halt || flushCtx.Err() != nil { + failedHalves = append(failedHalves, half) + continue + } + err := batchPushRefs(flushCtx, target, half) + if err == nil { + res.landed += len(half) + onLanded(half) + continue + } + if _, unreachable := remote.UnreachableRemoteLine(err); unreachable || nonInteractiveSSHAuthFailure(flushCtx, err) { + halt = true + } + logging.Debug(ctx, "git-refs push: split checkpoint ref batch failed", + slog.Int("refs", len(half)), slog.String("error", err.Error())) + failedHalves = append(failedHalves, half) + } + if len(failedHalves) == 1 && !halt { + walk(failedHalves[0]) + return + } + for _, half := range failedHalves { + res.unresolved = append(res.unresolved, half...) + } + } + for start := 0; start < len(refs); start += chunkSize { + walk(refs[start:min(start+chunkSize, len(refs))]) + } + return res +} + // chunkStopReason reports why the batch phase should stop before its next // chunk, or "" to continue. Same bare phrasing as flushAbortReason. func chunkStopReason(ctx context.Context, consecutiveFailures int) string { diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index 92200b76d3..d6cb3a7514 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -92,8 +92,9 @@ func TestFlushCheckpointRefs_StopsAfterConsecutiveFailures(t *testing.T) { output := restore() require.NoError(t, err, "a bounded flush must still never block the user's push") - // One batch attempt, then the fallback stops after the cap. - assert.Equal(t, 1+maxConsecutiveRefPushFailures, countedAttempts(t, countFile), + // One batch attempt, one split (both halves refused, so no deeper), then + // the fallback stops after the cap. + assert.Equal(t, 1+2+maxConsecutiveRefPushFailures, countedAttempts(t, countFile), "the fallback must not walk the whole queue against a refusing remote") assert.Contains(t, output, "Stopped retrying") assert.Contains(t, output, fmt.Sprintf("%d consecutive failures", maxConsecutiveRefPushFailures)) @@ -562,9 +563,10 @@ func TestPushQueuedCheckpointRefs_FallbackGetsFreshBudget(t *testing.T) { } // TestFlushCheckpointRefs_PartialDeliveryStillCountsAsDelivered: a chunked -// flush can land chunks and then fail (here, SSH auth on a later chunk). The -// landed chunks reached the remote, so the pre-push acts on them — capture, -// misdirection warning — rather than reading the error as "nothing synced". +// flush can land chunks and then fail (here, SSH auth on a later chunk); it +// reports the landed count alongside the error. That the pre-push then acts on +// the count is pinned end to end by +// TestCheckpointSyncRemote_PartialDeliveryCapturesElection. func TestFlushCheckpointRefs_PartialDeliveryStillCountsAsDelivered(t *testing.T) { shrinkRefPushChunkSize(t, 2) workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 4) @@ -619,3 +621,65 @@ func TestFlushCheckpointRefs_RejectionProvesRemoteReachable(t *testing.T) { assert.Equal(t, refHashOf(t, repo, refs[1]), remoteRefHash(t, bareDir, refs[1]), "the rejected chunk's healthy ref lands through the fallback") } + +// TestFlushCheckpointRefs_CutAfterProgressKeepsChunkSize: a budget cut after +// chunks landed is a backlog draining as intended, not a chunk too large for +// the link, so it must not shrink the chunk size — at one ref per push every +// push would pay the remote's whole ref advertisement. +func TestFlushCheckpointRefs_CutAfterProgressKeepsChunkSize(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 6) + countFile := filepath.Join(t.TempDir(), "attempts") + prepareGitRefsPrePush(t, workDir, bareDir) + installCountingHook(t, bareDir, countFile, 1) // first chunk lands, the rest stall + restoreBudget := checkpointFlushBudget + checkpointFlushBudget = 3 * time.Second + t.Cleanup(func() { checkpointFlushBudget = restoreBudget }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + require.Contains(t, output, "exhausted", "precondition: the budget cut the flush") + assert.Equal(t, 2, queue.ChunkSizeHint(checkpointRefPushChunkSize), "progress was made, so the size stays") +} + +// TestFlushCheckpointRefs_SplitIsolatesRejectedRef: a forge declines a whole +// push for one bad ref, failing its chunk. Halving finds that ref in a few +// pushes and lands the rest, instead of retrying the chunk one ref at a time. +func TestFlushCheckpointRefs_SplitIsolatesRejectedRef(t *testing.T) { + shrinkRefPushChunkSize(t, 16) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 16) + countFile := filepath.Join(t.TempDir(), "attempts") + prepareGitRefsPrePush(t, workDir, bareDir) + hook := "#!/bin/sh\necho attempt >> '" + countFile + "'\nblocked=0\nwhile read -r old new ref; do\n" + + " [ \"$ref\" = '" + refs[11].String() + "' ] && blocked=1\ndone\n" + + "if [ \"$blocked\" = 1 ]; then echo '" + checkpointRejectReason + "' >&2; exit 1; fi\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + restore() + + // 1 chunk, then 16 -> 8 -> 4 (floor): 2 pushes per level, then the 4 + // unresolved refs one at a time. + assert.Equal(t, 1+2+2+4, countedAttempts(t, countFile), "halving, not 16 single-ref retries") + for i, ref := range refs { + if i == 11 { + continue + } + assert.Equal(t, refHashOf(t, repo, ref), remoteRefHash(t, bareDir, ref)) + } + remaining, err := queue.Drain() + require.NoError(t, err) + assert.Equal(t, []plumbing.ReferenceName{refs[11]}, remaining, "only the rejected ref stays queued") +} diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index 491d3641a7..50bc97031f 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -102,7 +102,7 @@ Rewrites are atomic (temp file + rename under the lock) so a concurrent reader n 1. **Drains** the queue. 2. **Partitions** the drained refs into those that still exist locally and stale ones (dropped from the queue). 3. **Batch-pushes** the existing refs a chunk at a time, one network round-trip per `checkpointRefPushChunkSize` refs (`pushRefChunks` → `batchPushRefs`, `strategy/push_common.go`), **removing** each chunk from the queue as soon as it lands. -4. On a chunk failure (typically a non-fast-forward rejection), carries on with the next chunk, then **falls back to per-ref recovery** (`pushCheckpointRefWithRecovery`) for the failed chunks' refs only, removing from the queue only the refs that land. +4. On a chunk failure (typically a non-fast-forward rejection), carries on with the next chunk, then re-pushes each failed chunk **in halves** (`splitFailedChunks`) — a forge declines a whole push for one bad ref, so halving isolates it in about 2·log2(chunk) pushes and lands the rest — and finally **falls back to per-ref recovery** (`pushCheckpointRefWithRecovery`) for what is left, removing from the queue only the refs that land. Halving stops where both halves fail (the cause is not one ref) or at four refs. Confirmed remote policy/hook rejections skip fetch/replay: replay cannot fix a policy block and would needlessly rewrite local commits if the ref already exists @@ -118,7 +118,7 @@ without additional secret redaction of the remote's diagnostic output. Plain non SSH authentication failures retain their dedicated hint. Failures remain queued and **never fail the user's git push**. -The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch the budget cuts — between chunks or mid-chunk — ends the flush there ("Stopped pushing: budget … exhausted"); the fallback is not run on a spent budget, and the cut chunk rotates to the back of the queue. The chunk size adapts to the link (`PushQueue.ChunkSizeHint`, `entire-checkpoint-push-chunk-size` in the git common dir): a chunk the budget cut quarters it, down to one ref, and a batch that lands everything doubles it back toward `checkpointRefPushChunkSize` (200). The ceiling is high on purpose: every push pays a fixed cost — the connection plus the remote advertising every checkpoint ref it holds — so a healthy link is fastest in as few pushes as possible. Measured against GitHub and Entire, 200 checkpoints (~20MB) took 3–7s as one push and 11–20s in chunks of 25. Without that, a link too slow to land one full chunk within the budget would cut the same chunk on every push and never make progress. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it, and opens the fallback's budget only after the batch. Under the pre-push hook, OpenSSH also gets `ConnectTimeout=60` (`withBatchModeSSH`), so an unreachable host fails within a minute instead of spending the budget on a kernel connect timeout. As a command-line option it overrides a `ConnectTimeout` in `~/.ssh/config`; 60s leaves room for slow-starting proxies, whose banner wait it also bounds. Other ssh clients get only what they accept, mirroring git's own variant detection (`GIT_SSH_VARIANT`, `ssh.variant`, then the program name — `auto` defers to the name): plink gets `-batch`, its equivalent of `BatchMode`, never `-o`; TortoisePlink and the `simple` variant get nothing; an unrecognized wrapper keeps `BatchMode` only. A `GIT_SSH` program path is shell-quoted when rewritten into `GIT_SSH_COMMAND`, so a path with spaces stays one word. When the git config that names the ssh command cannot be read, the environment is left unchanged rather than replacing the user's command with a guess. +The batch is bounded too. Before chunking, one `git push` carried the whole queue with no deadline, so a large backlog over a slow uplink or a stalled connection held the user's `git push` for as long as the upload took. In the pre-push path the batch now shares the fallback's `checkpointFlushBudget`, and stops before its next chunk once that budget is spent or `maxConsecutiveChunkPushFailures` chunks fail in a row; it always attempts the first chunk. Chunks are what keep the bound from discarding work: landed chunks leave the queue at once, so a backlog larger than one budget drains over several pushes instead of being cut at the same point by each. A batch the budget cuts — between chunks or mid-chunk — ends the flush there ("Stopped pushing: budget … exhausted"); the fallback is not run on a spent budget, and the cut chunk rotates to the back of the queue. The chunk size adapts to the link (`PushQueue.ChunkSizeHint`, `entire-checkpoint-push-chunk-size` in the git common dir): a chunk the budget cut before anything landed quarters it, down to one ref (a cut after chunks landed is a backlog draining as intended and keeps the size), and a batch that lands everything doubles it back toward `checkpointRefPushChunkSize` (200). The ceiling is high on purpose: every push pays a fixed cost — the connection plus the remote advertising every checkpoint ref it holds — so a healthy link is fastest in as few pushes as possible. Measured against GitHub and Entire, 200 checkpoints (~20MB) took 3–7s as one push and 11–20s in chunks of 25. Without that, a link too slow to land one full chunk within the budget would cut the same chunk on every push and never make progress. The explicit migration push (`PushQueuedCheckpointRefs`) leaves the batch unbounded, because the user asked for that upload and is waiting on it, and opens the fallback's budget only after the batch. Under the pre-push hook, OpenSSH also gets `ConnectTimeout=60` (`withBatchModeSSH`), so an unreachable host fails within a minute instead of spending the budget on a kernel connect timeout. As a command-line option it overrides a `ConnectTimeout` in `~/.ssh/config`; 60s leaves room for slow-starting proxies, whose banner wait it also bounds. Other ssh clients get only what they accept, mirroring git's own variant detection (`GIT_SSH_VARIANT`, `ssh.variant`, then the program name — `auto` defers to the name): plink gets `-batch`, its equivalent of `BatchMode`, never `-o`; TortoisePlink and the `simple` variant get nothing; an unrecognized wrapper keeps `BatchMode` only. A `GIT_SSH` program path is shell-quoted when rewritten into `GIT_SSH_COMMAND`, so a path with spaces stays one word. When the git config that names the ssh command cannot be read, the environment is left unchanged rather than replacing the user's command with a guess. A remote that cannot be reached at all ends the flush after one attempt. When nothing has landed and a chunk's git output carries a connect-phase failure from ssh or curl (`remote.UnreachableRemoteLine`: unresolvable host, failed connect), later chunks and the per-ref fallback would each fail the same way after their own connect timeout, so the flush stops, prints that one line — credentials redacted, control bytes stripped, capped — and leaves the whole queue in order. The match is deliberately narrow: no bare strerror text (curl's is localized), no mid-transfer timeouts (the host was reachable), and no `remote:` lines (a reachable remote's own hook failing to resolve a host). A miss only costs the old path, the bounded fallback. From 127096d6299a1ddf18de9ee98469f31a91b383c8 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 20:26:19 -0700 Subject: [PATCH 08/16] fix(checkpoint): show an earlier refusal when the budget stops the flush When the budget cut the batch, every failed chunk went straight to the "stay queued" stop, including one the remote had refused outright earlier in the flush. The per-ref fallback that would show the remote's reason does not run on a spent budget, so a ref blocked for its content could ride along with every budget cut without ever saying why. Print the rejection reason from the stop itself. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4FB52FBYGR1B33N8QYAKBFS --- cmd/entire/cli/strategy/manual_commit_push.go | 7 +++++ .../cli/strategy/refs_push_bounds_test.go | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index 2fa1fac214..130bfcc658 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -706,6 +706,13 @@ func stopOnBudgetCut(ctx, flushCtx context.Context, queue *checkpoint.PushQueue, flushAbortReason(flushCtx, 0), queued-batch.landed) logging.Warn(ctx, "git-refs push: batch push cut by the flush budget; remaining refs stay queued", slog.Int("pushed", batch.landed), slog.Int("queued", queued)) + // An earlier chunk may have been refused outright before the clock ran + // out. The fallback that would show why does not run on a spent budget, + // so show the remote's reason here: a ref blocked for its content would + // otherwise ride along with every budget cut without ever saying so. + if reason := checkpointRefRejectionReason(batch.firstErr); reason != "" { + fmt.Fprintf(os.Stderr, "[entire] Warning: the remote declined a checkpoint ref batch (showing one rejection):\n%s\n", reason) + } // Fair scheduling, as after a fallback abort: a slow remote cuts the // budget at the same place every push, so the cut chunk goes to the // back rather than holding the head of the queue. diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index d6cb3a7514..d4fdbb57cc 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -683,3 +683,31 @@ func TestFlushCheckpointRefs_SplitIsolatesRejectedRef(t *testing.T) { require.NoError(t, err) assert.Equal(t, []plumbing.ReferenceName{refs[11]}, remaining, "only the rejected ref stays queued") } + +// TestFlushCheckpointRefs_BudgetCutStillShowsEarlierRejection: a chunk the +// remote refused before the budget ran out never reaches the per-ref fallback +// in that flush, so the budget stop itself must surface the refusal. +func TestFlushCheckpointRefs_BudgetCutStillShowsEarlierRejection(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 6) + prepareGitRefsPrePush(t, workDir, bareDir) + hook := "#!/bin/sh\nblocked=0\nwhile read -r old new ref; do\n" + + " [ \"$ref\" = '" + refs[0].String() + "' ] && blocked=1\ndone\n" + + "if [ \"$blocked\" = 1 ]; then echo '" + checkpointRejectReason + "' >&2; exit 1; fi\nsleep 30\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) + restoreBudget := checkpointFlushBudget + checkpointFlushBudget = 3 * time.Second + t.Cleanup(func() { checkpointFlushBudget = restoreBudget }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + require.Contains(t, output, "Stopped pushing", "precondition: the budget cut the flush") + assert.Contains(t, output, checkpointRejectReason, "the earlier refusal is still shown") +} From cd83ebad5d976de58bf380a545c4da6f381f02c9 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 16:07:14 -0700 Subject: [PATCH 09/16] feat(checkpoint): finish slow git-refs uploads in a background worker The pre-push hook now uploads queued checkpoint refs inline for a short budget (5s). An ordinary push finishes inside it and behaves as before. A queue that does not fit is handed to one detached `entire __checkpoint_upload` worker per repository, so the user's git push returns and the remaining checkpoints follow. Only a budget stop is handed off; unreachable, refusing or unauthenticated remotes stay queued for the next push as before. - All flushes serialise on an upload lock in the git common dir. A push that finds it held leaves a request; whoever releases the lock starts a worker for a waiting request. The migration push waits (bounded) and says so. - The hook resolves the OPF decision before locking and carries it, with the pending sync-remote capture, to the worker, which never re-resolves them, commits the capture only after delivery, and uploads nothing if settings or redaction cannot be loaded or a required OPF run cannot happen. - The worker records its last run; the next push prints a failure or announcement once, and `entire status` shows a running or stopped upload. - Uploads stay inline under ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND=1 (trail create, test harnesses), in CI, and with strategy_options.background_checkpoint_upload: false. - SpawnDetached gives children the null device instead of a pipe (ported from the OPF scan worker branch) so a worker survives its first stderr write after the parent exits. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4EWAPN9JDYSTB9X7NZK3P20 --- cmd/entire/cli/checkpoint/upload_state.go | 251 +++++++++++ .../cli/checkpoint/upload_state_test.go | 98 +++++ cmd/entire/cli/checkpoint_upload_cmd.go | 26 ++ cmd/entire/cli/execx/spawn_detached.go | 31 +- cmd/entire/cli/execx/spawn_detached_test.go | 25 ++ .../background_checkpoint_upload_test.go | 91 ++++ cmd/entire/cli/integration_test/setup_test.go | 4 + cmd/entire/cli/root.go | 1 + cmd/entire/cli/settings/settings.go | 11 + cmd/entire/cli/status.go | 22 + .../cli/strategy/checkpoint_upload_worker.go | 393 ++++++++++++++++++ .../strategy/checkpoint_upload_worker_test.go | 335 +++++++++++++++ cmd/entire/cli/strategy/manual_commit_push.go | 289 ++++++++++--- cmd/entire/cli/strategy/push_common.go | 19 +- .../cli/strategy/refs_push_bounds_test.go | 8 +- cmd/entire/cli/strategy/refs_push_test.go | 3 +- cmd/entire/cli/trail_cmd.go | 4 + docs/architecture/ref-checkpoint-backend.md | 14 + docs/development/checkpoint-implementation.md | 1 + e2e/controlplane/main_test.go | 1 + e2e/tests/main_test.go | 3 + 21 files changed, 1561 insertions(+), 69 deletions(-) create mode 100644 cmd/entire/cli/checkpoint/upload_state.go create mode 100644 cmd/entire/cli/checkpoint/upload_state_test.go create mode 100644 cmd/entire/cli/checkpoint_upload_cmd.go create mode 100644 cmd/entire/cli/execx/spawn_detached_test.go create mode 100644 cmd/entire/cli/integration_test/background_checkpoint_upload_test.go create mode 100644 cmd/entire/cli/strategy/checkpoint_upload_worker.go create mode 100644 cmd/entire/cli/strategy/checkpoint_upload_worker_test.go diff --git a/cmd/entire/cli/checkpoint/upload_state.go b/cmd/entire/cli/checkpoint/upload_state.go new file mode 100644 index 0000000000..3b6def7d5d --- /dev/null +++ b/cmd/entire/cli/checkpoint/upload_state.go @@ -0,0 +1,251 @@ +package checkpoint + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/go-git/go-git/v6" + + "github.com/entireio/cli/cmd/entire/cli/gitdir" + "github.com/entireio/cli/cmd/entire/cli/internal/flock" + "github.com/entireio/cli/cmd/entire/cli/jsonutil" +) + +// Background checkpoint upload coordination, kept in the git common dir next to +// the push queue: the queue is shared by every worktree, so is the one worker +// that drains it. +const ( + // uploadWorkerLockName is held for a whole flush of the queue — by the + // background worker for its run, and by every foreground flush — so two + // processes never push (and fetch+replay) the same refs at once. + uploadWorkerLockName = "entire-checkpoint-upload.lock" + uploadStateFileName = "entire-checkpoint-upload-state.json" + uploadStateLockName = "entire-checkpoint-upload-state.lock" +) + +// uploadWorkerTryWait turns the blocking flock into a try-lock: a caller that +// cannot take it almost at once treats the queue as being uploaded by its holder. +const uploadWorkerTryWait = 100 * time.Millisecond + +// OPF decisions a pre-push carries to the background worker. Unset means the +// pushing hook never resolved one; the worker then applies the non-interactive +// rules, never a skip. +const ( + UploadOPFUnset = "" + UploadOPFRun = "run" + UploadOPFSkip = "skip" +) + +// UploadRequest is a pre-push's hand-off to the background worker: what the +// hook decided in the foreground, where it can still ask the user, so the worker +// acts on that decision rather than re-deriving it later from changed state. +type UploadRequest struct { + Remote string `json:"remote"` + OPFDecision string `json:"opf_decision,omitempty"` + // PendingCapture is the checkpoint sync remote this push would elect. The + // worker persists it only once its own delivery lands, as the hook would. + PendingCapture string `json:"pending_capture,omitempty"` + RequestedAt time.Time `json:"requested_at"` +} + +// UploadResult is the background worker's record of its last run. +type UploadResult struct { + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at,omitzero"` + Pushed int `json:"pushed"` + Remaining int `json:"remaining"` + // Error is a one-line account of why refs stayed queued, already + // redacted for display; "" when the run left nothing behind. + Error string `json:"error,omitempty"` + // Announcement is a user-facing line the worker could not print itself + // (its stdio is the null device), shown by the next push and status. + Announcement string `json:"announcement,omitempty"` +} + +// Running reports whether the run had not finished as of now. A record older +// than maxRun is stale (the worker was killed), not running. +func (r *UploadResult) Running(now time.Time, maxRun time.Duration) bool { + return r != nil && r.FinishedAt.IsZero() && now.Sub(r.StartedAt) < maxRun +} + +// UploadState is the coordination file's contents. +type UploadState struct { + Request *UploadRequest `json:"request,omitempty"` + Last *UploadResult `json:"last,omitempty"` +} + +// UploadCoordinator owns the background upload's lock and state file for one +// git common dir. +type UploadCoordinator struct { + dir string +} + +// NewUploadCoordinator returns the coordinator rooted at gitCommonDir. +func NewUploadCoordinator(gitCommonDir string) *UploadCoordinator { + return &UploadCoordinator{dir: gitCommonDir} +} + +// UploadCoordinatorForRepo resolves the git common dir for repo. +func UploadCoordinatorForRepo(repo *git.Repository) (*UploadCoordinator, error) { + _, dir, err := repositoryDirs(repo) + if err != nil { + return nil, fmt.Errorf("resolve git common dir for checkpoint upload: %w", err) + } + return NewUploadCoordinator(dir), nil +} + +func (c *UploadCoordinator) root() (*os.Root, error) { + return gitdir.OpenAt(c.dir) //nolint:wrapcheck // gitdir names the directory; callers add context +} + +// TryLockWorker takes the upload lock if it is free. ok is false, with a nil +// error, when another process holds it. +func (c *UploadCoordinator) TryLockWorker(ctx context.Context) (release func(), ok bool, err error) { + lockCtx, cancel := context.WithTimeout(ctx, uploadWorkerTryWait) + defer cancel() + release, err = c.LockWorker(lockCtx) + if err != nil { + if errors.Is(lockCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil { + return nil, false, nil + } + return nil, false, err + } + return release, true, nil +} + +// LockWorker waits for the upload lock until ctx is done. +func (c *UploadCoordinator) LockWorker(ctx context.Context) (func(), error) { + root, err := c.root() + if err != nil { + return nil, fmt.Errorf("open git common dir: %w", err) + } + release, err := flock.AcquireContextIn(ctx, root, uploadWorkerLockName) + if err != nil { + return nil, fmt.Errorf("lock checkpoint upload: %w", err) + } + return release, nil +} + +// Request records req for the worker, merged with any request not yet taken: +// the latest push's destination wins, and so does a decision to run OPF — a +// later push that resolved nothing must not drop an earlier push's "run". +func (c *UploadCoordinator) Request(req UploadRequest) error { + return c.update(func(st *UploadState) { + if st.Request != nil && st.Request.OPFDecision == UploadOPFRun { + req.OPFDecision = UploadOPFRun + } + if req.PendingCapture == "" && st.Request != nil && st.Request.Remote == req.Remote { + req.PendingCapture = st.Request.PendingCapture + } + st.Request = &req + }) +} + +// TakeRequest removes and returns the pending request, or nil. +func (c *UploadCoordinator) TakeRequest() (*UploadRequest, error) { + var req *UploadRequest + err := c.update(func(st *UploadState) { + req, st.Request = st.Request, nil + }) + return req, err +} + +// StartRun records a run starting at started. An announcement no push has +// shown yet is carried into the new record rather than lost with the old one. +func (c *UploadCoordinator) StartRun(started time.Time) error { + return c.update(func(st *UploadState) { + res := UploadResult{StartedAt: started} + if st.Last != nil { + res.Announcement = st.Last.Announcement + } + st.Last = &res + }) +} + +// FinishRun records the outcome of the run StartRun began, appending its +// announcement to any still unshown. +func (c *UploadCoordinator) FinishRun(res UploadResult) error { + return c.update(func(st *UploadState) { + if st.Last != nil && st.Last.Announcement != "" { + res.Announcement = strings.TrimSpace(st.Last.Announcement + "\n" + res.Announcement) + } + st.Last = &res + }) +} + +// ClearReported drops the failure and announcement of the run that started at +// startedAt, once a push has shown them. A newer run's record is left alone. +func (c *UploadCoordinator) ClearReported(startedAt time.Time) error { + return c.update(func(st *UploadState) { + if st.Last != nil && st.Last.StartedAt.Equal(startedAt) { + st.Last.Error = "" + st.Last.Announcement = "" + } + }) +} + +// HasRequest reports whether a request is waiting for a worker. +func (c *UploadCoordinator) HasRequest() bool { + st, err := c.State() + return err == nil && st.Request != nil +} + +// State reads the coordination file. A missing file is the zero state. +func (c *UploadCoordinator) State() (UploadState, error) { + root, err := c.root() + if err != nil { + return UploadState{}, fmt.Errorf("open git common dir: %w", err) + } + release, err := flock.AcquireIn(root, uploadStateLockName) + if err != nil { + return UploadState{}, fmt.Errorf("lock checkpoint upload state: %w", err) + } + defer release() + return readUploadState(root) +} + +func (c *UploadCoordinator) update(mutate func(*UploadState)) error { + root, err := c.root() + if err != nil { + return fmt.Errorf("open git common dir: %w", err) + } + release, err := flock.AcquireIn(root, uploadStateLockName) + if err != nil { + return fmt.Errorf("lock checkpoint upload state: %w", err) + } + defer release() + st, err := readUploadState(root) + if err != nil { + return err + } + mutate(&st) + data, err := jsonutil.MarshalIndentWithNewline(st, "", " ") + if err != nil { + return fmt.Errorf("encode checkpoint upload state: %w", err) + } + if err := jsonutil.WriteFileAtomicIn(root, uploadStateFileName, data, 0o600); err != nil { + return fmt.Errorf("write checkpoint upload state: %w", err) + } + return nil +} + +func readUploadState(root *os.Root) (UploadState, error) { + var st UploadState + data, err := root.ReadFile(uploadStateFileName) + if errors.Is(err, os.ErrNotExist) { + return st, nil + } + if err != nil { + return st, fmt.Errorf("read checkpoint upload state: %w", err) + } + if err := json.Unmarshal(data, &st); err != nil { + // A corrupt file must not wedge uploads: start over. + return UploadState{}, nil //nolint:nilerr // deliberate reset of unreadable state + } + return st, nil +} diff --git a/cmd/entire/cli/checkpoint/upload_state_test.go b/cmd/entire/cli/checkpoint/upload_state_test.go new file mode 100644 index 0000000000..71a53f8139 --- /dev/null +++ b/cmd/entire/cli/checkpoint/upload_state_test.go @@ -0,0 +1,98 @@ +package checkpoint + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestUploadCoordinator_RequestMerge(t *testing.T) { + t.Parallel() + coord := NewUploadCoordinator(t.TempDir()) + + require.NoError(t, coord.Request(UploadRequest{Remote: "origin", OPFDecision: UploadOPFRun, PendingCapture: "origin"})) + require.NoError(t, coord.Request(UploadRequest{Remote: "origin", OPFDecision: UploadOPFUnset})) + + req, err := coord.TakeRequest() + require.NoError(t, err) + require.NotNil(t, req) + assert.Equal(t, UploadOPFRun, req.OPFDecision, "a later push that resolved nothing must not drop an earlier run") + assert.Equal(t, "origin", req.PendingCapture, "an unsatisfied capture for the same remote is kept") + + req, err = coord.TakeRequest() + require.NoError(t, err) + assert.Nil(t, req, "taking consumes the request") +} + +func TestUploadCoordinator_RequestToOtherRemoteDropsCapture(t *testing.T) { + t.Parallel() + coord := NewUploadCoordinator(t.TempDir()) + require.NoError(t, coord.Request(UploadRequest{Remote: "origin", PendingCapture: "origin"})) + require.NoError(t, coord.Request(UploadRequest{Remote: "fork"})) + + req, err := coord.TakeRequest() + require.NoError(t, err) + require.NotNil(t, req) + assert.Equal(t, "fork", req.Remote) + assert.Empty(t, req.PendingCapture) +} + +func TestUploadCoordinator_RunRecords(t *testing.T) { + t.Parallel() + coord := NewUploadCoordinator(t.TempDir()) + st, err := coord.State() + require.NoError(t, err) + assert.Nil(t, st.Last, "no file is the zero state") + + first := time.Now() + require.NoError(t, coord.StartRun(first)) + require.NoError(t, coord.FinishRun(UploadResult{StartedAt: first, FinishedAt: first, Announcement: "[entire] one"})) + second := first.Add(time.Second) + require.NoError(t, coord.StartRun(second)) + require.NoError(t, coord.FinishRun(UploadResult{StartedAt: second, FinishedAt: second, Remaining: 3, + Error: "budget (10m0s) exhausted", Announcement: "[entire] two"})) + st, err = coord.State() + require.NoError(t, err) + assert.Equal(t, "[entire] one\n[entire] two", st.Last.Announcement, "an unshown announcement survives the next run") + + require.NoError(t, coord.ClearReported(first)) + st, err = coord.State() + require.NoError(t, err) + assert.NotEmpty(t, st.Last.Error, "clearing an older run must not wipe a newer one") + + require.NoError(t, coord.ClearReported(second)) + st, err = coord.State() + require.NoError(t, err) + assert.Empty(t, st.Last.Error) + assert.Empty(t, st.Last.Announcement) + assert.Equal(t, 3, st.Last.Remaining) +} + +func TestUploadResult_Running(t *testing.T) { + t.Parallel() + now := time.Now() + assert.True(t, (&UploadResult{StartedAt: now.Add(-time.Minute)}).Running(now, time.Hour)) + assert.False(t, (&UploadResult{StartedAt: now.Add(-2 * time.Hour)}).Running(now, time.Hour), "a killed worker's record goes stale") + assert.False(t, (&UploadResult{StartedAt: now, FinishedAt: now}).Running(now, time.Hour)) + assert.False(t, (*UploadResult)(nil).Running(now, time.Hour)) +} + +func TestUploadCoordinator_TryLockWorker(t *testing.T) { + t.Parallel() + coord := NewUploadCoordinator(t.TempDir()) + release, ok, err := coord.TryLockWorker(t.Context()) + require.NoError(t, err) + require.True(t, ok) + + _, ok, err = coord.TryLockWorker(t.Context()) + require.NoError(t, err) + assert.False(t, ok, "a held lock is reported, not an error") + + release() + release2, ok, err := coord.TryLockWorker(t.Context()) + require.NoError(t, err) + assert.True(t, ok) + release2() +} diff --git a/cmd/entire/cli/checkpoint_upload_cmd.go b/cmd/entire/cli/checkpoint_upload_cmd.go new file mode 100644 index 0000000000..6e63e556ed --- /dev/null +++ b/cmd/entire/cli/checkpoint_upload_cmd.go @@ -0,0 +1,26 @@ +package cli + +import ( + "github.com/spf13/cobra" + + "github.com/entireio/cli/cmd/entire/cli/strategy" +) + +// newCheckpointUploadCmd creates the hidden command the pre-push hook spawns +// detached to finish a git-refs checkpoint upload that did not fit its inline +// budget (see strategy.RunCheckpointUploadWorker). Not for direct use. +func newCheckpointUploadCmd() *cobra.Command { + return &cobra.Command{ + Use: "__checkpoint_upload", + Hidden: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + // Detached child with its stdio on the null device: attach the file + // logger so a failed background upload is diagnosable in + // .entire/logs/entire.log. See newSweepSessionsCmd. + ensureLogger(cmd) + strategy.RunCheckpointUploadWorker(cmd.Context()) + return nil + }, + } +} diff --git a/cmd/entire/cli/execx/spawn_detached.go b/cmd/entire/cli/execx/spawn_detached.go index 07a6cc9f0b..846e225055 100644 --- a/cmd/entire/cli/execx/spawn_detached.go +++ b/cmd/entire/cli/execx/spawn_detached.go @@ -2,7 +2,6 @@ package execx import ( "context" - "io" "os" "os/exec" "testing" @@ -13,7 +12,7 @@ import ( // CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS on Windows, via detachFromTTY). // The child runs in dir (os.TempDir() when empty, so the child never holds the // parent's working directory), inherits the parent's environment, and has its -// stdout/stderr discarded. Best-effort: every error is swallowed — callers +// stdout/stderr sent to the null device. Best-effort: every error is swallowed — callers // treat the spawn as advisory background work. // // In-process `go test` runs are a no-op: the current executable is the test @@ -28,6 +27,24 @@ func SpawnDetached(dir string, args ...string) { return } + cmd := detachedCommand(executable, dir, args...) + if err := cmd.Start(); err != nil { + return + } + // Release the process so it can run independently of the parent. + //nolint:errcheck // best effort — the child continues regardless + _ = cmd.Process.Release() +} + +// detachedCommand builds the child SpawnDetached starts. Separate from the +// spawn so tests can check how the child is wired without forking it. +// +// Stdout and stderr stay nil, which os/exec opens as the null device. A +// non-*os.File writer such as io.Discard would instead hand the child a pipe +// drained by a goroutine in this process; once this process exits, the child's +// next write to that pipe raises SIGPIPE and kills it. A child that reports +// progress on stderr (`__opf_flush` does) would die on its first line. +func detachedCommand(executable, dir string, args ...string) *exec.Cmd { // context.Background(): the child must outlive the parent, so it is never // tied to a cancellable context. cmd := exec.CommandContext(context.Background(), executable, args...) @@ -37,13 +54,5 @@ func SpawnDetached(dir string, args ...string) { cmd.Dir = os.TempDir() } cmd.Env = os.Environ() - cmd.Stdout = io.Discard - cmd.Stderr = io.Discard - - if err := cmd.Start(); err != nil { - return - } - // Release the process so it can run independently of the parent. - //nolint:errcheck // best effort — the child continues regardless - _ = cmd.Process.Release() + return cmd } diff --git a/cmd/entire/cli/execx/spawn_detached_test.go b/cmd/entire/cli/execx/spawn_detached_test.go new file mode 100644 index 0000000000..27f4002b79 --- /dev/null +++ b/cmd/entire/cli/execx/spawn_detached_test.go @@ -0,0 +1,25 @@ +package execx + +import ( + "testing" +) + +// A detached child must not write into a pipe this process drains: after the +// parent exits, the child's next stdout/stderr write raises SIGPIPE and kills +// it. Nil streams are opened as the null device instead. +func TestDetachedCommand_StdioIsNullDevice(t *testing.T) { + t.Parallel() + cmd := detachedCommand("/bin/entire", "", "__opf_flush") + if cmd.Stdout != nil { + t.Errorf("Stdout = %T; want nil so the child writes to the null device, not a pipe", cmd.Stdout) + } + if cmd.Stderr != nil { + t.Errorf("Stderr = %T; want nil so the child writes to the null device, not a pipe", cmd.Stderr) + } + if cmd.Stdin != nil { + t.Errorf("Stdin = %T; want nil", cmd.Stdin) + } + if cmd.Dir == "" { + t.Error("Dir is empty; want os.TempDir() fallback") + } +} diff --git a/cmd/entire/cli/integration_test/background_checkpoint_upload_test.go b/cmd/entire/cli/integration_test/background_checkpoint_upload_test.go new file mode 100644 index 0000000000..4c13499c74 --- /dev/null +++ b/cmd/entire/cli/integration_test/background_checkpoint_upload_test.go @@ -0,0 +1,91 @@ +//go:build integration + +package integration + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/execx" + "github.com/entireio/cli/cmd/entire/cli/strategy" +) + +// TestGitPushWithHooks_BackgroundUploadFinishesSlowPush runs the hybrid end to +// end through the real binary: the pre-push hook's inline upload is cut by its +// budget on a slow remote, the hook spawns the detached worker and returns, and +// the worker lands the checkpoint after the user's push has finished. +func TestGitPushWithHooks_BackgroundUploadFinishesSlowPush(t *testing.T) { + t.Parallel() + + env := NewFeatureBranchEnv(t) + env.CheckpointStore = StoreGitRefs + bareDir := env.SetupBareRemote() + checkpointID := createCheckpointedCommit(t, env, "Add auth module", "auth.go", "package auth", "Add auth module") + if checkpointID == "" { + t.Fatal("should have a checkpoint ID after condensation") + } + + // The first push carrying checkpoint refs outlasts the inline budget; every + // later push (the user's branch, the worker's upload) is immediate. + marker := filepath.Join(t.TempDir(), "slowed") + hook := "#!/bin/sh\nif grep -q refs/entire/ && [ ! -e '" + marker + "' ]; then touch '" + marker + "'; sleep 30; fi\nexit 0\n" + if err := os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755); err != nil { + t.Fatal(err) + } + + env.InstallRealPrePushHook() + cmd := execx.NonInteractive(t.Context(), "git", "push", "origin", "HEAD") + cmd.Dir = env.RepoDir + // Opt this one push back into the hand-off the harness disables, and out + // of the CI detection a CI runner would trip. + cmd.Env = append(env.cliEnv(), strategy.CheckpointUploadForegroundEnv+"=", + "CI=", "GITHUB_ACTIONS=", "GITLAB_CI=", "BUILDKITE=", "JENKINS_URL=", "TF_BUILD=") + start := time.Now() + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git push failed: %v\n%s", err, output) + } + if elapsed := time.Since(start); elapsed > 20*time.Second { + t.Errorf("git push took %s; the hook must hand off instead of waiting out the slow remote", elapsed) + } + if !strings.Contains(string(output), "in the background") { + t.Fatalf("hook did not hand off to the background worker:\n%s", output) + } + + // Wait for the worker to finish before the test's temp dirs go away. + commonDir := filepath.Join(env.RepoDir, ".git") + coord := checkpoint.NewUploadCoordinator(commonDir) + deadline := time.Now().Add(90 * time.Second) + var last *checkpoint.UploadResult + for time.Now().Before(deadline) { + if st, stErr := coord.State(); stErr == nil && st.Last != nil && !st.Last.FinishedAt.IsZero() { + last = st.Last + break + } + time.Sleep(200 * time.Millisecond) + } + if last == nil { + t.Fatal("background worker did not finish within 90s") + } + // The worker still releases its lock and logs after recording the run; + // let it exit before the test's temp dirs are removed. + for time.Now().Before(deadline) { + release, ok, lockErr := coord.TryLockWorker(t.Context()) + if lockErr == nil && ok { + release() + break + } + time.Sleep(100 * time.Millisecond) + } + time.Sleep(time.Second) + if last.Error != "" { + t.Fatalf("background worker left refs queued: %s", last.Error) + } + if !env.CheckpointExistsOnRemote(bareDir, checkpointID) { + t.Fatalf("checkpoint %s should be on the remote after the background upload", checkpointID) + } +} diff --git a/cmd/entire/cli/integration_test/setup_test.go b/cmd/entire/cli/integration_test/setup_test.go index 86368da35d..c4a19fccc5 100644 --- a/cmd/entire/cli/integration_test/setup_test.go +++ b/cmd/entire/cli/integration_test/setup_test.go @@ -15,6 +15,7 @@ import ( "github.com/entireio/cli/cmd/entire/cli/agent" "github.com/entireio/cli/cmd/entire/cli/execx" + "github.com/entireio/cli/cmd/entire/cli/strategy" "github.com/entireio/cli/cmd/entire/cli/testutil" "github.com/entireio/cli/internal/entireclient/contexts" ) @@ -61,6 +62,9 @@ func TestMain(m *testing.M) { // security allowlist should not be widened for test convenience. // testing.Testing() is false in the spawned binary, so it is told here. "ENTIRE_TEST_ALLOW_FILE_REMOTES": "1", + // Keep checkpoint uploads inside the pre-push hook: a detached + // background upload would outlive the test and its temp remotes. + strategy.CheckpointUploadForegroundEnv: "1", } for k, v := range isolation { if err := os.Setenv(k, v); err != nil { diff --git a/cmd/entire/cli/root.go b/cmd/entire/cli/root.go index 978bcb9515..720e965cb5 100644 --- a/cmd/entire/cli/root.go +++ b/cmd/entire/cli/root.go @@ -230,6 +230,7 @@ func NewRootCmd() *cobra.Command { cmd.AddCommand(newCurlBashPostInstallCmd()) cmd.AddCommand(newRefreshTrailEnablementCmd()) cmd.AddCommand(newSweepSessionsCmd()) + cmd.AddCommand(newCheckpointUploadCmd()) // Experimental command (developer-only visibility; setup/tune runners). experimental.Register(cmd, newRunnerCmd()) // 'runner' (experimental) diff --git a/cmd/entire/cli/settings/settings.go b/cmd/entire/cli/settings/settings.go index 19621200e7..1e0ec6326d 100644 --- a/cmd/entire/cli/settings/settings.go +++ b/cmd/entire/cli/settings/settings.go @@ -2116,6 +2116,17 @@ func (s *EntireSettings) IsPushSessionsDisabled() bool { return false } +// IsBackgroundCheckpointUploadDisabled reports whether +// strategy_options.background_checkpoint_upload is explicitly false, which keeps +// the whole git-refs checkpoint upload inside the pre-push hook. +func (s *EntireSettings) IsBackgroundCheckpointUploadDisabled() bool { + if s.StrategyOptions == nil { + return false + } + val, ok := s.StrategyOptions["background_checkpoint_upload"].(bool) + return ok && !val +} + // IsExternalAgentsEnabled checks if external agent discovery is enabled in settings. // Returns false by default if settings cannot be loaded or the key is missing. func IsExternalAgentsEnabled(ctx context.Context) bool { diff --git a/cmd/entire/cli/status.go b/cmd/entire/cli/status.go index 4fb40c58d4..abc0272fe1 100644 --- a/cmd/entire/cli/status.go +++ b/cmd/entire/cli/status.go @@ -407,6 +407,11 @@ type checkpointSyncInfo struct { // when none, when counting failed, or when the count would be a lie // (dedicated URL mode on the git-branch backend). Unpushed int + // UploadRunning and UploadLastError report the background checkpoint + // upload (git-refs): one in progress now, or why the last one stopped + // short with refs still queued. + UploadRunning bool + UploadLastError string // IgnoredRemote and IgnoredReason report a configured checkpoint_remote // that the ownership check rejected as inherited with the clone. Both // reads and pushes then fall back to the elected remote, and status is @@ -543,6 +548,7 @@ func computeCheckpointSyncInfo(ctx context.Context, s *EntireSettings) checkpoin // unpushed". if cpCfg, cfgErr := settings.LoadCheckpointsConfig(ctx); cfgErr == nil && checkpoint.PrimaryIsRefs(cpCfg) { info.Unpushed = countUnpushedCheckpointsForStatus(ctx, "") + info.UploadRunning, info.UploadLastError = strategy.CheckpointUploadStatus(ctx) } return info } @@ -551,6 +557,7 @@ func computeCheckpointSyncInfo(ctx context.Context, s *EntireSettings) checkpoin info.Remote = elected.Name info.Source = string(elected.Source) info.Unpushed = countUnpushedCheckpointsForStatus(ctx, elected.Name) + info.UploadRunning, info.UploadLastError = strategy.CheckpointUploadStatus(ctx) // A configured checkpoint_remote that did not enable above is being // ignored. When the ownership check is what rejected it, say so: this is // the one trust-gate rejection a user otherwise experiences only as @@ -679,6 +686,10 @@ func writeCheckpointSyncLines(ctx context.Context, b *strings.Builder, s *Entire b.WriteString("\n ") b.WriteString(sty.render(sty.dim, formatUnpushedCheckpointsLine(info))) } + if info.Unpushed > 0 && info.UploadLastError != "" { + b.WriteString("\n") + b.WriteString(sty.render(sty.yellow, " ! Last background checkpoint upload stopped: "+info.UploadLastError)) + } } // formatUnpushedCheckpointsLine phrases the unpushed counter. Dedicated URL @@ -708,6 +719,9 @@ func formatUnpushedCheckpointsLine(info checkpointSyncInfo) string { } return fmt.Sprintf("%d %s not on %s", info.Unpushed, noun, info.Remote) } + if info.UploadRunning { + return fmt.Sprintf("%d %s uploading in the background", info.Unpushed, noun) + } if info.Source == checkpointSyncSourceDedicated { return fmt.Sprintf("%d %s not yet pushed", info.Unpushed, noun) } @@ -1149,6 +1163,10 @@ type statusJSON struct { // not mean reads fall open to nothing. CheckpointReadFallback string `json:"checkpoint_read_fallback,omitempty"` UnpushedCheckpoints int `json:"unpushed_checkpoints,omitempty"` + // CheckpointUploadRunning/-LastError mirror the text path's background + // upload lines (git-refs). + CheckpointUploadRunning bool `json:"checkpoint_upload_running,omitempty"` + CheckpointUploadLastError string `json:"checkpoint_upload_last_error,omitempty"` // CheckpointRemoteIgnored/-Reason report a configured checkpoint_remote the // ownership check rejected as inherited with the clone (reads and pushes // fall back to the elected remote). Mirrors the text path's warning line. @@ -1267,6 +1285,10 @@ func runStatusJSON(ctx context.Context, w io.Writer) error { result.CheckpointReadFallback = syncInfo.ReadFallback result.CheckpointReadSourceUnknown = syncInfo.ReadSourceUnknown result.UnpushedCheckpoints = syncInfo.Unpushed + result.CheckpointUploadRunning = syncInfo.UploadRunning + if syncInfo.Unpushed > 0 { + result.CheckpointUploadLastError = syncInfo.UploadLastError + } result.CheckpointRemoteIgnored = syncInfo.IgnoredRemote result.CheckpointRemoteIgnoredReason = syncInfo.IgnoredReason result.CheckpointRemoteIgnoredRemedy = syncInfo.IgnoredRemedy diff --git a/cmd/entire/cli/strategy/checkpoint_upload_worker.go b/cmd/entire/cli/strategy/checkpoint_upload_worker.go new file mode 100644 index 0000000000..eed74a04ef --- /dev/null +++ b/cmd/entire/cli/strategy/checkpoint_upload_worker.go @@ -0,0 +1,393 @@ +// Background checkpoint upload (git-refs). +// +// The pre-push hook uploads queued checkpoint refs inline for a short budget +// (checkpointInlineUploadBudget). A queue that does not fit — a backlog, a slow +// uplink — is handed to one detached `entire __checkpoint_upload` worker per +// repository, so the user's `git push` returns in seconds and the remaining +// checkpoints follow. An ordinary push finishes inside the inline budget and +// behaves exactly as before: checkpoints are on the remote when the push returns. +// +// The worker acts only on what the pushing hook decided in the foreground, where +// the user can still be asked: the OPF decision and the pending sync-remote +// capture travel in the request (checkpoint.UploadRequest). Its stdio is the +// null device, so what it would have printed is recorded instead +// (checkpoint.UploadResult) and shown by the next push and by `entire status`. +package strategy + +import ( + "bytes" + "context" + "fmt" + "log/slog" + "os" + "strings" + "testing" + "time" + + "github.com/go-git/go-git/v6" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + checkpointremote "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" + "github.com/entireio/cli/cmd/entire/cli/execx" + "github.com/entireio/cli/cmd/entire/cli/gitrepo" + "github.com/entireio/cli/cmd/entire/cli/logging" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/settings" + "github.com/entireio/cli/redact" +) + +const checkpointUploadComponent = "checkpoint-upload" + +// CheckpointUploadForegroundEnv, when set to 1, keeps the whole upload inside +// the pre-push hook. Set by callers that report on the hook's delivery +// (`entire trail create`) and by test harnesses whose temp remotes vanish when +// the test ends. +const CheckpointUploadForegroundEnv = "ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND" + +// checkpointInlineUploadBudget is how long the pre-push hook uploads before it +// hands the rest to the worker. Long enough that an ordinary push never hands +// off — its checkpoints are on the remote when the push returns, which +// entire.io, `entire trail create`, and short-lived environments rely on — and +// short enough that a backlog no longer holds the user's push. Var for tests. +var checkpointInlineUploadBudget = 5 * time.Second + +// foregroundUploadLockWait bounds how long a foreground flush (hand-off +// disabled) waits for a running worker before leaving its refs queued. +const foregroundUploadLockWait = 30 * time.Second + +const ( + // checkpointUploadDeliveryBudget bounds one worker pass's flush. + checkpointUploadDeliveryBudget = 10 * time.Minute + // checkpointUploadWorkerDeadline bounds a worker's whole run, and is how + // long a run record without a finish time counts as still running. + checkpointUploadWorkerDeadline = 30 * time.Minute + // checkpointUploadMaxPasses bounds the passes a worker makes for requests + // that arrive while it runs. + checkpointUploadMaxPasses = 5 +) + +// ciEnvVars mark environments whose machine or checkout is discarded when the +// job ends, taking a detached worker's queue with it: upload inline there. +var ciEnvVars = []string{"CI", "GITHUB_ACTIONS", "GITLAB_CI", "BUILDKITE", "JENKINS_URL", "TF_BUILD"} + +// backgroundUploadInTests lets a test opt into the hand-off; in-process tests +// otherwise upload inline, as before. +var backgroundUploadInTests = false + +// checkpointUploadSpawn is the process-spawn seam, swapped in tests: +// execx.SpawnDetached is a no-op under `go test`. +var checkpointUploadSpawn = func(worktreeRoot string) { + execx.SpawnDetached(worktreeRoot, "__checkpoint_upload") +} + +type uploadWorkerKey struct{} + +// withinUploadWorker marks ctx as the background worker, which uploads inline +// and never hands off or spawns another worker. +func withinUploadWorker(ctx context.Context) context.Context { + return context.WithValue(ctx, uploadWorkerKey{}, true) +} + +func inUploadWorker(ctx context.Context) bool { + v, _ := ctx.Value(uploadWorkerKey{}).(bool) //nolint:errcheck // absent means false + return v +} + +type carriedOPFDecisionKey struct{} + +func withCarriedOPFDecision(ctx context.Context, decision string) context.Context { + return context.WithValue(ctx, carriedOPFDecisionKey{}, decision) +} + +// carriedOPFDecision is the OPF decision a hand-off carried, or UploadOPFUnset. +func carriedOPFDecision(ctx context.Context) string { + v, _ := ctx.Value(carriedOPFDecisionKey{}).(string) //nolint:errcheck // absent means unset + return v +} + +// backgroundCheckpointUploadEnabled reports whether this pre-push may hand the +// rest of its upload to the background worker. +func backgroundCheckpointUploadEnabled(ctx context.Context) bool { + if inUploadWorker(ctx) { + return false + } + if testing.Testing() && !backgroundUploadInTests { + return false + } + if os.Getenv(CheckpointUploadForegroundEnv) == "1" { + return false + } + for _, name := range ciEnvVars { + if os.Getenv(name) != "" { + return false + } + } + if s, err := settings.Load(ctx); err == nil && s.IsBackgroundCheckpointUploadDisabled() { + return false + } + return true +} + +// lockQueueForFlush takes the upload lock for a pre-push flush, so it never +// pushes the same refs as a running worker. With hand-off enabled it only tries: +// a running worker already owns the queue, and the caller records a request for +// it instead. Otherwise it waits up to foregroundUploadLockWait. ok is false +// when the lock was not taken; a coordinator that cannot be used at all must not +// block uploads, so that returns ok with a no-op release. +func lockQueueForFlush(ctx context.Context, coord *checkpoint.UploadCoordinator, background bool) (release func(), ok bool) { + if coord == nil { + return func() {}, true + } + if background { + release, ok, err := coord.TryLockWorker(ctx) + if err != nil { + logging.Debug(ctx, "checkpoint upload lock unavailable; uploading without it", + slog.String("error", err.Error())) + return func() {}, true + } + return release, ok + } + lockCtx, cancel := context.WithTimeout(ctx, foregroundUploadLockWait) + defer cancel() + release, err := coord.LockWorker(lockCtx) + if err != nil { + if lockCtx.Err() != nil && ctx.Err() == nil { + return nil, false + } + logging.Debug(ctx, "checkpoint upload lock unavailable; uploading without it", + slog.String("error", err.Error())) + return func() {}, true + } + return release, true +} + +// reportBackgroundUpload prints, once, what the last worker run could not: a +// failure that left refs queued, and any line it would have printed. +func reportBackgroundUpload(ctx context.Context, coord *checkpoint.UploadCoordinator) { + if coord == nil { + return + } + st, err := coord.State() + if err != nil || st.Last == nil || st.Last.Running(time.Now(), checkpointUploadWorkerDeadline) { + return + } + if st.Last.Announcement == "" && st.Last.Error == "" { + return + } + if st.Last.Announcement != "" { + fmt.Fprintln(stderrWriter, st.Last.Announcement) + } + if st.Last.Error != "" { + fmt.Fprintf(stderrWriter, "[entire] Last background checkpoint upload stopped: %s.\n", st.Last.Error) + } + if clearErr := coord.ClearReported(st.Last.StartedAt); clearErr != nil { + logging.Debug(ctx, "clear checkpoint upload result failed", slog.String("error", clearErr.Error())) + } +} + +// handOffCheckpointUpload records req for the worker and starts one. The caller +// must have released the upload lock, or the new worker exits at once — which +// is safe: whoever holds the lock starts a worker for the request on release +// (releaseUploadLock). +func handOffCheckpointUpload(ctx context.Context, coord *checkpoint.UploadCoordinator, req checkpoint.UploadRequest) bool { + req.RequestedAt = time.Now() + if err := coord.Request(req); err != nil { + logging.Warn(ctx, "record background checkpoint upload request failed", + slog.String("error", err.Error())) + return false + } + spawnCheckpointUploadWorker(ctx) + logging.Info(logging.WithComponent(ctx, checkpointUploadComponent), + "handed checkpoint upload to background worker", slog.String("remote", req.Remote)) + return true +} + +func spawnCheckpointUploadWorker(ctx context.Context) { + root, err := paths.WorktreeRoot(ctx) + if err != nil { + logging.Debug(ctx, "skipping background checkpoint upload spawn: no worktree root", + slog.String("error", err.Error())) + return + } + checkpointUploadSpawn(root) +} + +// releaseUploadLock releases a flush's hold on the upload lock and, if a push +// recorded a request while it was held, starts a worker for it. A push that +// found the lock held relied on its holder to get the request done; a worker +// it spawned exited at once, since the lock was taken. +func releaseUploadLock(ctx context.Context, coord *checkpoint.UploadCoordinator, release func()) { + release() + if coord != nil && !inUploadWorker(ctx) && coord.HasRequest() { + spawnCheckpointUploadWorker(ctx) + } +} + +// RunCheckpointUploadWorker is the body of the detached +// `entire __checkpoint_upload` worker. Each pass takes the pending request and +// uploads the queue under it; another pass runs only for a request that arrived +// meanwhile, and none after a pass that left refs queued — the next push +// retries those. Best-effort: failures are recorded and logged, never +// returned, because nothing watches this process's exit code. +func RunCheckpointUploadWorker(ctx context.Context) { + ctx = logging.WithComponent(ctx, checkpointUploadComponent) + defer logging.Info(ctx, "checkpoint upload worker finished") + // A detached child inherits the hook's environment, which may pin GIT_DIR + // and friends to the spawning worktree; resolve from the working directory + // instead, like any other CWD-based command. + unsetRepoOverrideEnv() + ctx = checkpointremote.WithNonInteractiveSSH(withinUploadWorker(ctx)) + ctx, cancel := context.WithTimeout(ctx, checkpointUploadWorkerDeadline) + defer cancel() + + repo, err := OpenRepository(ctx) + if err != nil { + logging.Warn(ctx, "checkpoint upload worker: open repository failed", slog.String("error", err.Error())) + return + } + defer repo.Close() + coord, err := checkpoint.UploadCoordinatorForRepo(repo) + if err != nil { + logging.Warn(ctx, "checkpoint upload worker: resolve state failed", slog.String("error", err.Error())) + return + } + // OPFEnabled reads process-global config only this sets. Without it the + // gate reads "OPF off" and ships unscanned content, so a worker that cannot + // configure redaction uploads nothing. EnsureRedactionConfigured survives a + // settings read failure (the hook path must), so check that separately. + _, settingsErr := settings.Load(ctx) + if err := EnsureRedactionConfigured(ctx); err != nil || settingsErr != nil { + now := time.Now() + if startErr := coord.StartRun(now); startErr == nil { + recordUploadFinish(ctx, coord, checkpoint.UploadResult{StartedAt: now, FinishedAt: time.Now(), + Error: "settings could not be loaded, so nothing was uploaded"}) + } + return + } + + // A pass that leaves refs queued ends the run (the next push retries them) + // unless a push asked for more meanwhile; passes are bounded either way. + for range checkpointUploadMaxPasses { + release, ok, lockErr := coord.TryLockWorker(ctx) + if lockErr != nil || !ok { + return // The holder starts a worker for any request on release. + } + req, takeErr := coord.TakeRequest() + if takeErr != nil || req == nil { + release() + return + } + runCheckpointUploadPass(ctx, repo, coord, *req) + release() + if !coord.HasRequest() { + return + } + } +} + +// runCheckpointUploadPass uploads the queue for req and records the outcome. +func runCheckpointUploadPass(ctx context.Context, repo *git.Repository, coord *checkpoint.UploadCoordinator, req checkpoint.UploadRequest) { + result := checkpoint.UploadResult{StartedAt: time.Now()} + if err := coord.StartRun(result.StartedAt); err != nil { + logging.Warn(ctx, "record checkpoint upload start failed", slog.String("error", err.Error())) + } + finish := func(errLine string) { + result.FinishedAt = time.Now() + result.Error = errLine + recordUploadFinish(ctx, coord, result) + } + + // Lines the hook path writes to the user (the capture announcement, the + // ignored-checkpoint_remote warning) have no terminal here: collect them + // for the next push to show. + var printed bytes.Buffer + restore := stderrWriter + stderrWriter = &printed + defer func() { stderrWriter = restore }() + + ctx = withCarriedOPFDecision(ctx, req.OPFDecision) + ps := resolvePushSettings(ctx, req.Remote) + if ps.pushDisabled { + finish("") + return + } + if !ps.hasCheckpointURL() && !checkpointSyncAllowedForRemote(ctx, ps.remote, req.PendingCapture) { + finish("checkpoints no longer sync to " + ps.remote) + return + } + // The hook decided OPF must run, but this process sees it off (another + // worktree's settings, or a scanner that failed to configure): uploading + // now would ship exactly what the user asked to have scanned. + if req.OPFDecision == checkpoint.UploadOPFRun && !redact.OPFEnabled() { + finish("the privacy filter could not run here, so nothing was uploaded") + return + } + if opfErr := opfGateForCheckpointRefs(ctx, repo); opfErr != nil { + logging.Warn(ctx, "background checkpoint upload withheld", slog.String("error", opfErr.Error())) + finish("withheld before upload (see .entire/logs)") + return + } + + deliverCtx, cancel := context.WithTimeout(ctx, checkpointUploadDeliveryBudget) + defer cancel() + res, err := flushCheckpointRefsQueue(deliverCtx, repo, ps, flushOptions{budget: checkpointUploadDeliveryBudget}) + if err != nil { + logging.Warn(ctx, "background checkpoint upload: refs left queued", slog.String("error", err.Error())) + } + if res.pushed > 0 { + if req.PendingCapture != "" { + commitCapturedSyncRemote(ctx, req.PendingCapture) + } + warnIgnoredCheckpointRemote(ctx, ps) + } + result.Pushed, result.Remaining = res.pushed, res.remaining + result.Announcement = strings.TrimSpace(printed.String()) + finish(res.failureLine(err)) +} + +func recordUploadFinish(ctx context.Context, coord *checkpoint.UploadCoordinator, res checkpoint.UploadResult) { + if err := coord.FinishRun(res); err != nil { + logging.Warn(ctx, "record checkpoint upload result failed", slog.String("error", err.Error())) + } +} + +// unsetRepoOverrideEnv removes the git repo-selector variables +// gitrepo.EnvWithoutRepoOverrides filters, from this process's environment. +func unsetRepoOverrideEnv() { + keep := make(map[string]bool) + for _, kv := range gitrepo.EnvWithoutRepoOverrides() { + keep[kv] = true + } + for _, kv := range os.Environ() { + if keep[kv] { + continue + } + if name, _, ok := strings.Cut(kv, "="); ok { + os.Unsetenv(name) + } + } +} + +// CheckpointUploadStatus reports, for `entire status`, whether a background +// checkpoint upload is running and why the last one stopped short ("" when it +// did not). Best-effort: any error reads as no background upload. +func CheckpointUploadStatus(ctx context.Context) (running bool, lastError string) { + repo, err := OpenRepository(ctx) + if err != nil { + return false, "" + } + defer repo.Close() + coord, err := checkpoint.UploadCoordinatorForRepo(repo) + if err != nil { + return false, "" + } + st, err := coord.State() + if err != nil || st.Last == nil { + return false, "" + } + running = st.Last.Running(time.Now(), checkpointUploadWorkerDeadline) + if running { + return true, "" + } + return false, st.Last.Error +} diff --git a/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go b/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go new file mode 100644 index 0000000000..d0a7ae8094 --- /dev/null +++ b/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go @@ -0,0 +1,335 @@ +package strategy + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-git/go-git/v6" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/gitrepo" +) + +// enableBackgroundUpload opts this test into the hand-off, which in-process +// tests otherwise skip, clears the CI markers a CI runner sets, and records +// spawns instead of forking. Not parallel-safe: it swaps package state. +func enableBackgroundUpload(t *testing.T, inlineBudget time.Duration) *[]string { + t.Helper() + for _, name := range ciEnvVars { + t.Setenv(name, "") + } + t.Setenv(CheckpointUploadForegroundEnv, "") + restoreAllowed, restoreSpawn, restoreBudget := backgroundUploadInTests, checkpointUploadSpawn, checkpointInlineUploadBudget + var spawned []string + backgroundUploadInTests = true + checkpointUploadSpawn = func(root string) { spawned = append(spawned, root) } + checkpointInlineUploadBudget = inlineBudget + t.Cleanup(func() { + backgroundUploadInTests, checkpointUploadSpawn, checkpointInlineUploadBudget = restoreAllowed, restoreSpawn, restoreBudget + }) + return &spawned +} + +func uploadCoordinator(t *testing.T, repo *git.Repository) *checkpoint.UploadCoordinator { + t.Helper() + coord, err := checkpoint.UploadCoordinatorForRepo(repo) + require.NoError(t, err) + return coord +} + +// TestPrePush_HandsBacklogToBackgroundWorker is the hybrid: the hook uploads +// what fits its inline budget, then hands the rest to one background worker +// instead of holding the user's push — and says so, rather than "stay queued". +func TestPrePush_HandsBacklogToBackgroundWorker(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 6) + countFile := filepath.Join(t.TempDir(), "attempts") + prepareGitRefsPrePush(t, workDir, bareDir) + installCountingHook(t, bareDir, countFile, 1) // first chunk lands, the rest stall + spawned := enableBackgroundUpload(t, 5*time.Second) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + start := time.Now() + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + elapsed := time.Since(start) + output := restore() + + assert.Less(t, elapsed, 25*time.Second, "the hook must stop at its inline budget, not wait out the stall") + assert.Len(t, *spawned, 1, "one background worker for the remainder") + assert.Contains(t, output, "Uploading the remaining 4 checkpoint ref(s) in the background") + assert.NotContains(t, output, "Stopped pushing", "a handed-off stop is not reported as left for the next push") + + remaining, err := queue.Drain() + require.NoError(t, err) + assert.ElementsMatch(t, refs[2:], remaining, "the landed chunk left the queue; the worker gets the rest") + st, err := uploadCoordinator(t, repo).State() + require.NoError(t, err) + require.NotNil(t, st.Request, "the hand-off is recorded for the worker") + assert.Equal(t, "origin", st.Request.Remote) +} + +// TestPrePush_NoHandOffWhenRemoteUnreachable: only a budget stop is handed off. +// A worker would fail against an unreachable remote exactly as the hook did. +func TestPrePush_NoHandOffWhenRemoteUnreachable(t *testing.T) { + workDir, _, refs := setupRepoWithNCheckpointRefs(t, 3) + prepareGitRefsPrePush(t, workDir, "ssh://git@example.invalid/repo.git") + fakeSSH := filepath.Join(t.TempDir(), "ssh") + require.NoError(t, os.WriteFile(fakeSSH, + []byte("#!/bin/sh\necho 'ssh: connect to host example.invalid port 22: Connection refused' >&2\nexit 255\n"), 0o755)) + t.Setenv("GIT_SSH_COMMAND", fakeSSH) + spawned := enableBackgroundUpload(t, 5*time.Second) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + assert.Empty(t, *spawned) + assert.Contains(t, output, "Couldn't reach origin") + assert.NotContains(t, output, "in the background") +} + +// TestPrePush_RunningWorkerOwnsTheQueue: while a worker holds the upload lock, +// the hook neither pushes the same refs alongside it nor waits for it — it +// leaves a request the worker picks up after its current pass. +func TestPrePush_RunningWorkerOwnsTheQueue(t *testing.T) { + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 3) + countFile := filepath.Join(t.TempDir(), "attempts") + prepareGitRefsPrePush(t, workDir, bareDir) + installCountingHook(t, bareDir, countFile, 0) + enableBackgroundUpload(t, 5*time.Second) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + coord := uploadCoordinator(t, repo) + release, err := coord.LockWorker(t.Context()) + require.NoError(t, err) + defer release() + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + assert.Equal(t, 0, countedAttempts(t, countFile), "nothing pushed alongside the running worker") + assert.Contains(t, output, "already running in the background") + remaining, err := queue.Drain() + require.NoError(t, err) + assert.ElementsMatch(t, refs, remaining) + st, err := coord.State() + require.NoError(t, err) + require.NotNil(t, st.Request) +} + +// TestPrePush_ForegroundEnvKeepsUploadInline: `entire trail create` and the +// test harnesses set the foreground env; then nothing is handed off and the +// pre-push budget applies as before. +func TestPrePush_ForegroundEnvKeepsUploadInline(t *testing.T) { + spawned := enableBackgroundUpload(t, time.Second) + t.Setenv(CheckpointUploadForegroundEnv, "1") + assert.False(t, backgroundCheckpointUploadEnabled(t.Context())) + t.Setenv(CheckpointUploadForegroundEnv, "") + t.Setenv("CI", "true") + assert.False(t, backgroundCheckpointUploadEnabled(t.Context()), "CI checkouts are discarded with the job") + t.Setenv("CI", "") + assert.False(t, backgroundCheckpointUploadEnabled(withinUploadWorker(t.Context())), "the worker never hands off") + assert.Empty(t, *spawned) +} + +// TestRunCheckpointUploadWorker_DrainsQueueAndRecords runs the worker body +// in-process against a healthy remote. +func TestRunCheckpointUploadWorker_DrainsQueueAndRecords(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 5) + prepareGitRefsPrePush(t, workDir, bareDir) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + coord := uploadCoordinator(t, repo) + require.NoError(t, coord.Request(checkpoint.UploadRequest{Remote: "origin"})) + + RunCheckpointUploadWorker(t.Context()) + + for _, ref := range refs { + assert.Equal(t, refHashOf(t, repo, ref), remoteRefHash(t, bareDir, ref)) + } + remaining, err := queue.Drain() + require.NoError(t, err) + assert.Empty(t, remaining) + st, err := coord.State() + require.NoError(t, err) + assert.Nil(t, st.Request, "the request was consumed") + require.NotNil(t, st.Last) + assert.False(t, st.Last.FinishedAt.IsZero()) + assert.Equal(t, len(refs), st.Last.Pushed) + assert.Empty(t, st.Last.Error) +} + +// TestRunCheckpointUploadWorker_FailureReachesNextPush: the worker has no +// terminal, so a run that left refs queued is recorded and the next push says +// so once. +func TestRunCheckpointUploadWorker_FailureReachesNextPush(t *testing.T) { + queued := maxConsecutiveRefPushFailures + 1 + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, queued) + prepareGitRefsPrePush(t, workDir, bareDir) + installCheckpointRejectHook(t, bareDir, false, filepath.Join(t.TempDir(), "attempts")) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + coord := uploadCoordinator(t, repo) + require.NoError(t, coord.Request(checkpoint.UploadRequest{Remote: "origin"})) + + restore := captureStderr(t) + RunCheckpointUploadWorker(t.Context()) + restore() + + st, err := coord.State() + require.NoError(t, err) + require.NotNil(t, st.Last) + assert.NotEmpty(t, st.Last.Error) + running, lastErr := CheckpointUploadStatus(t.Context()) + assert.False(t, running) + assert.Equal(t, st.Last.Error, lastErr) + + buf := captureStderrWriter(t) + restore = captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + restore() + assert.Contains(t, buf.String(), "Last background checkpoint upload stopped: "+st.Last.Error) + + st, err = coord.State() + require.NoError(t, err) + assert.Empty(t, st.Last.Error, "reported once") +} + +// TestRunCheckpointUploadWorker_ExitsWhenLockHeld: one worker per repository. +func TestRunCheckpointUploadWorker_ExitsWhenLockHeld(t *testing.T) { + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 2) + prepareGitRefsPrePush(t, workDir, bareDir) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + coord := uploadCoordinator(t, repo) + require.NoError(t, coord.Request(checkpoint.UploadRequest{Remote: "origin"})) + release, err := coord.LockWorker(t.Context()) + require.NoError(t, err) + defer release() + + RunCheckpointUploadWorker(t.Context()) + + remaining, err := queue.Drain() + require.NoError(t, err) + assert.ElementsMatch(t, refs, remaining, "the lock holder owns the queue") + st, err := coord.State() + require.NoError(t, err) + assert.NotNil(t, st.Request, "the request stays for the holder") +} + +// TestPushQueuedCheckpointRefs_WaitsForBackgroundWorker: the explicit +// migration push never pushes alongside a running worker. +func TestPushQueuedCheckpointRefs_WaitsForBackgroundWorker(t *testing.T) { + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 2) + prepareGitRefsPrePush(t, workDir, bareDir) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + release, err := uploadCoordinator(t, repo).LockWorker(t.Context()) + require.NoError(t, err) + defer release() + + ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond) + defer cancel() + _, _, err = PushQueuedCheckpointRefs(ctx, repo, "origin") + require.ErrorContains(t, err, "wait for background checkpoint upload") + assertRefsAbsentFromRemote(t, bareDir, refs, "nothing pushed while the worker held the queue") +} + +// TestOPFPrePushDecision_HonorsCarriedDecision: the worker acts on the hook's +// decision and never re-resolves it. +func TestOPFPrePushDecision_HonorsCarriedDecision(t *testing.T) { + t.Parallel() + got, err := opfPrePushDecision(withCarriedOPFDecision(t.Context(), checkpoint.UploadOPFSkip)) + require.NoError(t, err) + assert.Equal(t, OPFSkip, got) + got, err = opfPrePushDecision(withCarriedOPFDecision(t.Context(), checkpoint.UploadOPFRun)) + require.NoError(t, err) + assert.Equal(t, OPFRun, got) +} + +// TestReleaseUploadLock_StartsWorkerForWaitingRequest: a push that found the +// lock held left a request and relied on the holder; a holder that is not a +// worker must start one on release, or the request waits for the next push. +func TestReleaseUploadLock_StartsWorkerForWaitingRequest(t *testing.T) { + workDir, _, _ := setupRepoWithNCheckpointRefs(t, 1) + prepareGitRefsPrePush(t, workDir, t.TempDir()) + spawned := enableBackgroundUpload(t, 5*time.Second) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + coord := uploadCoordinator(t, repo) + + release, err := coord.LockWorker(t.Context()) + require.NoError(t, err) + releaseUploadLock(t.Context(), coord, release) + assert.Empty(t, *spawned, "no request, no worker") + + release, err = coord.LockWorker(t.Context()) + require.NoError(t, err) + require.NoError(t, coord.Request(checkpoint.UploadRequest{Remote: "origin"})) + releaseUploadLock(t.Context(), coord, release) + assert.Len(t, *spawned, 1) + + release, err = coord.LockWorker(t.Context()) + require.NoError(t, err) + releaseUploadLock(withinUploadWorker(t.Context()), coord, release) + assert.Len(t, *spawned, 1, "the worker loops over requests itself") +} + +// TestRunCheckpointUploadWorker_WithholdsWhenRequiredOPFIsOff: the hook +// decided OPF runs; a worker that sees OPF off must not ship the queue as is. +func TestRunCheckpointUploadWorker_WithholdsWhenRequiredOPFIsOff(t *testing.T) { + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 2) + prepareGitRefsPrePush(t, workDir, bareDir) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + coord := uploadCoordinator(t, repo) + require.NoError(t, coord.Request(checkpoint.UploadRequest{Remote: "origin", OPFDecision: checkpoint.UploadOPFRun})) + + RunCheckpointUploadWorker(t.Context()) + + assertRefsAbsentFromRemote(t, bareDir, refs, "nothing ships unscanned") + remaining, err := queue.Drain() + require.NoError(t, err) + assert.ElementsMatch(t, refs, remaining) + st, err := coord.State() + require.NoError(t, err) + require.NotNil(t, st.Last) + assert.Contains(t, st.Last.Error, "privacy filter") +} diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index 130bfcc658..ff659d7723 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "strings" + "time" git "github.com/go-git/go-git/v6" "github.com/go-git/go-git/v6/plumbing" @@ -228,6 +229,14 @@ func (s *ManualCommitStrategy) prePush(ctx context.Context, remote string, prote // settings > prompt > non-TTY auto-run). Shared by both checkpoint backends so // the precedence cannot drift between them. func opfPrePushDecision(ctx context.Context) (OPFDecision, error) { + // The background upload worker acts on the decision the pushing hook made, + // where the user could still be asked; it never re-asks or reinterprets it. + switch carriedOPFDecision(ctx) { + case checkpoint.UploadOPFRun: + return OPFRun, nil + case checkpoint.UploadOPFSkip: + return OPFSkip, nil + } cfg, _ := settings.Load(ctx) //nolint:errcheck // Load already failed at hook init; fall back to nil var opfCfg *settings.OPFSettings if cfg != nil && cfg.Redaction != nil { @@ -411,17 +420,61 @@ func (s *ManualCommitStrategy) prePushCheckpointRefs(ctx context.Context, ps pus } defer repo.Close() + coord, coordErr := checkpoint.UploadCoordinatorForRepo(repo) + if coordErr != nil { + logging.Debug(ctx, "git-refs pre-push: checkpoint upload state unavailable; uploading inline", + slog.String("error", coordErr.Error())) + coord = nil + } + background := coord != nil && backgroundCheckpointUploadEnabled(ctx) + reportBackgroundUpload(ctx, coord) + // OPF backend divergence: both paths fail closed, but this one does it // without blocking the user. The v1 path aborts the user's git push; here a // checkpoint-ref failure must never do that (see this function's doc), so // failing closed means withholding the flush — nothing un-OPF'd ships, the // refs stay queued, and the user's push proceeds. + // + // The decision is resolved before the upload lock is taken: it may prompt, + // and no other flush should wait on a user. The gate below and any + // background hand-off then act on it without asking again. + decision, opfErr := opfPrePushDecisionForHandOff(ctx) + if opfErr != nil { + warnOPFCheckpointRefsWithheld(ctx, opfErr) + return nil + } + ctx = withCarriedOPFDecision(ctx, decision) + + // One flush of the queue at a time. When another holds it — a worker, or a + // push or migration that starts one on release (releaseUploadLock) — leave + // a request instead of pushing the same refs alongside it. + release, locked := lockQueueForFlush(ctx, coord, background) + if !locked { + if handOffCheckpointUpload(ctx, coord, checkpoint.UploadRequest{Remote: ps.remote, + OPFDecision: decision, PendingCapture: pendingCapture}) { + fmt.Fprintln(os.Stderr, "[entire] A checkpoint upload is already running in the background; these checkpoints will follow it.") + } + return nil + } + released := false + releaseLock := func() { + if !released { + released = true + releaseUploadLock(ctx, coord, release) + } + } + defer releaseLock() + if opfErr := opfGateForCheckpointRefs(ctx, repo); opfErr != nil { warnOPFCheckpointRefsWithheld(ctx, opfErr) return nil } - flushed, err := flushCheckpointRefsQueue(ctx, repo, ps, true) + opts := flushOptions{boundBatch: true} + if background { + opts.budget, opts.handoff = checkpointInlineUploadBudget, true + } + res, err := flushCheckpointRefsQueue(ctx, repo, ps, opts) if err != nil { // Fail-soft: a checkpoint-ref push failure must never block the user's // git push. The refs stay queued for the next pre-push. @@ -434,15 +487,54 @@ func (s *ManualCommitStrategy) prePushCheckpointRefs(ctx context.Context, ps pus // queue pushed nothing, so it must not move the election or announce that // it had, nor warn that checkpoints were misdirected (see // warnIgnoredCheckpointRemote). - if flushed > 0 { + if res.pushed > 0 { if pendingCapture != "" { commitCapturedSyncRemote(ctx, pendingCapture) } warnIgnoredCheckpointRemote(ctx, ps) } + + // The inline budget ran out with refs still queued: the worker continues + // from here instead of the next push. Only a budget stop is handed off — a + // remote that is refusing or unreachable fails the worker the same way. + if background && res.budgetExhausted && res.remaining > 0 { + capture := pendingCapture + if res.pushed > 0 { + capture = "" // Already committed above. + } + releaseLock() + if handOffCheckpointUpload(ctx, coord, checkpoint.UploadRequest{Remote: ps.remote, + OPFDecision: decision, PendingCapture: capture}) { + fmt.Fprintf(os.Stderr, "[entire] Uploading the remaining %d checkpoint ref(s) in the background.\n", res.remaining) + } else { + fmt.Fprintf(os.Stderr, "[entire] %d checkpoint ref(s) stay queued for the next push.\n", res.remaining) + } + } return nil } +// opfPrePushDecisionForHandOff resolves the OPF decision for a push whose +// upload a running worker will carry out, without running OPF here: the worker +// rewrites under the decision. Abort is returned as an error, like the gate. +func opfPrePushDecisionForHandOff(ctx context.Context) (string, error) { + if !redact.OPFEnabled() { + return checkpoint.UploadOPFUnset, nil + } + decision, err := opfPrePushDecision(ctx) + if err != nil { + return checkpoint.UploadOPFUnset, err + } + switch decision { + case OPFAbort: + return checkpoint.UploadOPFUnset, ErrOPFAbortedByUser + case OPFSkip: + return checkpoint.UploadOPFSkip, nil + case OPFRun: + return checkpoint.UploadOPFRun, nil + } + return checkpoint.UploadOPFUnset, nil +} + // PushQueuedCheckpointRefs pushes any queued checkpoint refs to the configured // checkpoint remote, surfacing errors (unlike the fail-soft pre-push path); the // caller owns the repo. It returns the number of refs pushed and whether @@ -456,6 +548,21 @@ func PushQueuedCheckpointRefs(ctx context.Context, repo *git.Repository, remote if ps.pushDisabled { return 0, true, nil } + // Wait out a running background worker rather than push the same refs + // alongside it: the user asked for this upload and is waiting on it. + if coord, coordErr := checkpoint.UploadCoordinatorForRepo(repo); coordErr == nil { + release, ok, lockErr := coord.TryLockWorker(ctx) + if lockErr == nil && !ok { + fmt.Fprintln(os.Stderr, "[entire] Waiting for the background checkpoint upload to finish...") + waitCtx, cancel := context.WithTimeout(ctx, checkpointUploadDeliveryBudget) + defer cancel() + release, lockErr = coord.LockWorker(waitCtx) + } + if lockErr != nil { + return 0, false, fmt.Errorf("wait for background checkpoint upload: %w", lockErr) + } + defer releaseUploadLock(ctx, coord, release) + } if opfErr := opfGateForCheckpointRefs(ctx, repo); opfErr != nil { // Names no cause, matching flushCheckpointRefsQueue's retry message // below: the gate fails on an unresolvable decision or a failed scan, @@ -464,8 +571,8 @@ func PushQueuedCheckpointRefs(ctx context.Context, repo *git.Repository, remote // wrong problem. The wrapped error says which it was. return 0, false, fmt.Errorf("checkpoint refs stay queued: %w", opfErr) } - pushed, err = flushCheckpointRefsQueue(ctx, repo, ps, false) - return pushed, false, err + res, err := flushCheckpointRefsQueue(ctx, repo, ps, flushOptions{}) + return res.pushed, false, err } // flushCheckpointRefsQueue drains the push-discovery queue and batch-pushes the @@ -473,23 +580,22 @@ func PushQueuedCheckpointRefs(ctx context.Context, repo *git.Repository, remote // removing from the queue only the refs that land. It returns the number pushed. // // Shared by the git-refs pre-push path (which logs and ignores the error to -// never block the user's push) and the migration command's opt-in push (which -// surfaces it). boundBatch puts the batch push under checkpointFlushBudget too; -// the pre-push path sets it, the explicit migration push does not, and its -// per-ref fallback then gets a fresh budget after the batch. Stale entries — -// refs no longer present locally — are pruned so they don't block the queue -// forever. -func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps pushSettings, boundBatch bool) (int, error) { +// never block the user's push), the background upload worker, and the +// migration command's opt-in push (which surfaces it). See flushOptions for how +// each bounds it; an unbounded batch's per-ref fallback gets a fresh budget +// after the batch. Stale entries — refs no longer present locally — are pruned so +// they don't block the queue forever. +func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps pushSettings, opts flushOptions) (flushResult, error) { queue, err := checkpoint.PushQueueForRepo(ctx, repo) if err != nil { - return 0, fmt.Errorf("resolve push queue: %w", err) + return flushResult{}, fmt.Errorf("resolve push queue: %w", err) } queued, err := queue.Drain() if err != nil { - return 0, fmt.Errorf("drain push queue: %w", err) + return flushResult{}, fmt.Errorf("drain push queue: %w", err) } if len(queued) == 0 { - return 0, nil + return flushResult{}, nil } pushCtx, pushSpan := perf.Start(ctx, "push_checkpoint_refs") @@ -503,7 +609,7 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push } } if len(existing) == 0 { - return 0, nil + return flushResult{}, nil } // Resolved here, not by the caller: it spawns `git remote get-url` and its @@ -524,12 +630,21 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push // see checkpointFlushBudget and maxConsecutiveRefPushFailures. An unbounded // batch opens it after the batch instead, so its fallback is not left with // a budget the batch already spent. - flushCtx, cancelFlush := context.WithTimeout(pushCtx, checkpointFlushBudget) + budget := opts.budget + if budget <= 0 { + budget = checkpointFlushBudget + } + flushCtx, cancelFlush := context.WithTimeout(withFlushBudget(pushCtx, budget), budget) batchCtx := flushCtx - if !boundBatch { + if !opts.boundBatch { cancelFlush() batchCtx = pushCtx } + // The budget, not an interruption, ended the flush early: the only stop a + // handoff can continue in the background. + budgetExhausted := func() bool { + return errors.Is(flushCtx.Err(), context.DeadlineExceeded) && pushCtx.Err() == nil + } // Fast path: push the refs a chunk per round-trip (fast-forward-only). chunkSize := queue.ChunkSizeHint(checkpointRefPushChunkSize) @@ -539,42 +654,14 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push slog.String("error", removeErr.Error())) } }) - if !boundBatch { - flushCtx, cancelFlush = context.WithTimeout(pushCtx, checkpointFlushBudget) + if !opts.boundBatch { + flushCtx, cancelFlush = context.WithTimeout(withFlushBudget(pushCtx, budget), budget) } defer cancelFlush() - budgetCut := errors.Is(flushCtx.Err(), context.DeadlineExceeded) && pushCtx.Err() == nil + budgetCut := budgetExhausted() adaptChunkSize(ctx, queue, chunkSize, batch, budgetCut) - if len(batch.failed) == 0 && len(batch.untried) == 0 { - stop(" done") - return batch.landed, nil - } - if len(batch.failed) == 0 { - // Every attempted chunk landed, but the budget ran out first. Nothing - // failed, so this is progress rather than an error. - stop(fmt.Sprintf(" pushed %d of %d", batch.landed, len(existing))) - fmt.Fprintf(os.Stderr, "[entire] Stopped pushing: %s; %d checkpoint ref(s) stay queued for the next push.\n", - batch.stopReason, len(batch.untried)) - logging.Warn(ctx, "git-refs push: batch push stopped early; remaining refs stay queued", - slog.String("reason", batch.stopReason), slog.Int("pushed", batch.landed), - slog.Int("queued", len(existing))) - return batch.landed, nil - } - // The budget ran out mid-chunk: the per-ref fallback would only fail its - // one guaranteed attempt on the spent budget, reporting a retry that never - // ran as a push failure. It is the same stop as running out between chunks. - if budgetCut && !batch.sshAuthFailed && batch.unreachable == "" { - stopOnBudgetCut(ctx, flushCtx, queue, batch, len(existing), stop) - return batch.landed, nil - } - if batch.unreachable != "" { - stop(" failed") - } else { - stop("") - } - - if reportUnpushableDestination(dest, batch, len(existing)) { - return batch.landed, batch.firstErr + if res, done, err := settleBatchOnly(ctx, flushCtx, queue, batch, len(existing), dest, opts, stop, budgetCut); done { + return res, err } // At least one chunk failed — typically a non-fast-forward divergence (the @@ -611,7 +698,9 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push } // Printed before the rejection warning so the more specific reason lands // closest to the prompt. - if abortReason != "" { + res := flushResult{pushed: totalPushed, remaining: len(existing) - totalPushed, + budgetExhausted: len(existing) > totalPushed && budgetExhausted(), stopReason: abortReason} + if abortReason != "" && (!opts.handoff || !res.budgetExhausted) { // Everything this flush did not land stays queued, not just the refs it // never reached: only landed refs are removed, so the ones that were // attempted and failed are still there too. @@ -657,10 +746,104 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push // Counts attempts, not the queue: refs skipped by an early abort were // never tried, and reporting them as failures would overstate what the // remote actually refused. - return totalPushed, fmt.Errorf("%d of %d attempted checkpoint refs failed to push: %w", + return res, fmt.Errorf("%d of %d attempted checkpoint refs failed to push: %w", attempted-len(pushed), attempted, firstErr) } - return totalPushed, nil + return res, nil +} + +// settleBatchOnly finishes a flush whose batch phase decided its outcome on its +// own — everything landed, the budget cut it, or the destination refused the +// connection or the SSH key — printing that outcome. done is false when failed +// chunks remain for the per-ref fallback, which then owns the progress line. +func settleBatchOnly(ctx, flushCtx context.Context, queue *checkpoint.PushQueue, batch chunkPushResult, queued int, dest refsPushDestination, + opts flushOptions, stop func(string), budgetExhausted bool, +) (res flushResult, done bool, err error) { + if len(batch.failed) == 0 && len(batch.untried) == 0 { + stop(" done") + return flushResult{pushed: batch.landed}, true, nil + } + if len(batch.failed) == 0 { + // Every attempted chunk landed, but the budget ran out first. Nothing + // failed, so this is progress rather than an error. + stop(fmt.Sprintf(" pushed %d of %d", batch.landed, queued)) + res := flushResult{pushed: batch.landed, remaining: len(batch.untried), + budgetExhausted: budgetExhausted, stopReason: batch.stopReason} + if !opts.handoff || !res.budgetExhausted { + fmt.Fprintf(os.Stderr, "[entire] Stopped pushing: %s; %d checkpoint ref(s) stay queued for the next push.\n", + batch.stopReason, len(batch.untried)) + } + logging.Warn(ctx, "git-refs push: batch push stopped early; remaining refs stay queued", + slog.String("reason", batch.stopReason), slog.Int("pushed", batch.landed), + slog.Int("queued", queued)) + return res, true, nil + } + // The budget ran out mid-chunk: the per-ref fallback would only fail its + // one guaranteed attempt on the spent budget, reporting a retry that never + // ran as a push failure. With a hand-off the caller continues in the + // background and reports it; otherwise it is the same stop as running out + // between chunks. A destination that refused the key or the connection keeps + // its hint below instead. + if budgetExhausted && !batch.sshAuthFailed && batch.unreachable == "" { + if opts.handoff { + stop(fmt.Sprintf(" pushed %d of %d", batch.landed, queued)) + } else { + stopOnBudgetCut(ctx, flushCtx, queue, batch, queued, stop) + } + return flushResult{pushed: batch.landed, remaining: queued - batch.landed, + budgetExhausted: true, stopReason: flushAbortReason(flushCtx, 0)}, true, nil + } + if batch.unreachable != "" { + stop(" failed") + } else { + stop("") + } + if reportUnpushableDestination(dest, batch, queued) { + res := flushResult{pushed: batch.landed, remaining: queued - batch.landed} + if batch.sshAuthFailed { + res.stopReason = "SSH authentication failed" + } else { + res.unreachable = fmt.Sprintf("couldn't reach %s: %s", dest.display(), batch.unreachable) + } + return res, true, batch.firstErr + } + return flushResult{}, false, nil +} + +// flushOptions bounds one flushCheckpointRefsQueue. +type flushOptions struct { + // budget bounds the per-ref fallback, and the batch too when boundBatch is + // set; 0 means checkpointFlushBudget. + budget time.Duration + boundBatch bool + // handoff marks a flush whose budget stop the caller continues in the + // background: that stop is reported by the caller, not as "stay queued". + handoff bool +} + +// flushResult is what one flushCheckpointRefsQueue did, for callers that act on +// or record it rather than only print it. +type flushResult struct { + pushed int + remaining int // refs the flush attempted or skipped that are still queued + // budgetExhausted: the flush stopped because its budget ran out, not + // because the caller was interrupted. + budgetExhausted bool + stopReason string // bare phrase, as printed; "" when nothing stopped it + unreachable string // "couldn't reach : " when git could not connect +} + +// failureLine is a one-line account of a flush that left refs queued, or "". +func (r flushResult) failureLine(err error) string { + switch { + case r.unreachable != "": + return r.unreachable + case r.stopReason != "": + return r.stopReason + case err != nil: + return "checkpoint ref push failed" + } + return "" } // reportUnpushableDestination prints the hint for a batch that failed because diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index bbc3d8ce5b..54a1a5dd40 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -375,6 +375,23 @@ var checkpointPushBudget = 2 * time.Minute // Declared as a var so tests can shrink it. var checkpointFlushBudget = 2 * time.Minute +type flushBudgetKey struct{} + +// withFlushBudget records the budget a flush runs under, so its stop reason +// names that budget rather than the pre-push default. +func withFlushBudget(ctx context.Context, budget time.Duration) context.Context { + return context.WithValue(ctx, flushBudgetKey{}, budget) +} + +// flushBudgetFrom returns the budget withFlushBudget recorded, or +// checkpointFlushBudget. +func flushBudgetFrom(ctx context.Context) time.Duration { + if d, ok := ctx.Value(flushBudgetKey{}).(time.Duration); ok { + return d + } + return checkpointFlushBudget +} + // checkpointRefPushChunkSize is the most queued refs one batch push carries. // Large on purpose: every push pays a fixed cost — the connection and the // remote advertising every ref it holds, one per checkpoint — so a healthy @@ -416,7 +433,7 @@ const maxConsecutiveRefPushFailures = 5 func flushAbortReason(ctx context.Context, consecutiveFailures int) string { switch { case errors.Is(ctx.Err(), context.DeadlineExceeded): - return fmt.Sprintf("budget (%s) exhausted", checkpointFlushBudget) + return fmt.Sprintf("budget (%s) exhausted", flushBudgetFrom(ctx)) case ctx.Err() != nil: return "interrupted" case consecutiveFailures >= maxConsecutiveRefPushFailures: diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index d4fdbb57cc..bf76963bc7 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -446,7 +446,8 @@ func TestPushQueuedCheckpointRefs_BatchNotBudgeted(t *testing.T) { queue := enqueueRefs(t, repo, refs) restore := captureStderr(t) - pushed, err := flushCheckpointRefsQueue(t.Context(), repo, pushSettings{remote: bareDir}, false) + res, err := flushCheckpointRefsQueue(t.Context(), repo, pushSettings{remote: bareDir}, flushOptions{}) + pushed := res.pushed restore() require.NoError(t, err) assert.Equal(t, len(refs), pushed) @@ -555,7 +556,7 @@ func TestPushQueuedCheckpointRefs_FallbackGetsFreshBudget(t *testing.T) { enqueueRefs(t, repo, refs) restore := captureStderr(t) - _, err = flushCheckpointRefsQueue(t.Context(), repo, pushSettings{remote: bareDir}, false) + _, err = flushCheckpointRefsQueue(t.Context(), repo, pushSettings{remote: bareDir}, flushOptions{}) restore() require.Error(t, err, "the blocked ref still fails") assert.Equal(t, refHashOf(t, repo, refs[1]), remoteRefHash(t, bareDir, refs[1]), @@ -580,7 +581,8 @@ func TestFlushCheckpointRefs_PartialDeliveryStillCountsAsDelivered(t *testing.T) enqueueRefs(t, repo, refs) restore := captureStderr(t) - pushed, err := flushCheckpointRefsQueue(checkpointremote.WithNonInteractiveSSH(t.Context()), repo, pushSettings{remote: bareDir}, true) + res, err := flushCheckpointRefsQueue(checkpointremote.WithNonInteractiveSSH(t.Context()), repo, pushSettings{remote: bareDir}, flushOptions{boundBatch: true}) + pushed := res.pushed output := restore() require.Error(t, err) assert.Equal(t, 2, pushed, "the first chunk landed before the auth failure and is reported as pushed") diff --git a/cmd/entire/cli/strategy/refs_push_test.go b/cmd/entire/cli/strategy/refs_push_test.go index 1cd4703f3d..d5567298c6 100644 --- a/cmd/entire/cli/strategy/refs_push_test.go +++ b/cmd/entire/cli/strategy/refs_push_test.go @@ -166,7 +166,8 @@ func TestPushCheckpointRefWithRecovery_MergesDivergedRef(t *testing.T) { // pushes are rejected, then recovery replays C3's delta onto C2. queue := enqueueRefs(t, repo, []plumbing.ReferenceName{ref}) restore := captureStderr(t) - pushed, pushErr := flushCheckpointRefsQueue(ctx, repo, pushSettings{remote: bareDir}, true) + res, pushErr := flushCheckpointRefsQueue(ctx, repo, pushSettings{remote: bareDir}, flushOptions{boundBatch: true}) + pushed := res.pushed output := restore() require.NoError(t, pushErr, "diverged ref should be recovered by fetch+replay, not rejected") assert.Equal(t, 1, pushed) diff --git a/cmd/entire/cli/trail_cmd.go b/cmd/entire/cli/trail_cmd.go index 4cde2a3048..28fa93d7ac 100644 --- a/cmd/entire/cli/trail_cmd.go +++ b/cmd/entire/cli/trail_cmd.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "net/url" + "os" "os/exec" "reflect" "strconv" @@ -2501,6 +2502,9 @@ func pushBranchToRemote(ctx context.Context, out, errOut io.Writer, remote, bran cmd := exec.CommandContext(ctx, "git", "push", "-u", remote, branchName) cmd.Stdout = out cmd.Stderr = errOut + // Keep the hook's whole checkpoint upload in this push, never handed to the + // background worker: "Pushed branch" must mean the checkpoints went too. + cmd.Env = append(os.Environ(), strategy.CheckpointUploadForegroundEnv+"=1") if err := cmd.Run(); err != nil { // git's own message already reached errOut, so this names the command // and carries the exit status rather than repeating the diagnostic. diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index 50bc97031f..927127aae4 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -124,6 +124,18 @@ A remote that cannot be reached at all ends the flush after one attempt. When no The fallback is bounded. It runs under a `checkpointFlushBudget` context deadline — so an expired budget cuts a ref that is already hung rather than waiting out its own `checkpointPushBudget` first — and additionally stops after `maxConsecutiveRefPushFailures` refs fail in a row. Either way it never aborts before at least one ref has been attempted. A remote that is refusing or unreachable fails every ref the same way, and each ref in the fallback costs at least one network round-trip, so walking a large queue to the end turns one failed push into an apparently hung one for minutes or hours. Skipped refs stay queued and go out on the next push, and a success resets the consecutive count so one blocked checkpoint does not strand the refs queued behind it. The abort prints one line naming what stopped the retry and how many refs remain. +### Background upload (hybrid) + +The pre-push hook uploads inline for only `checkpointInlineUploadBudget` (5s). An ordinary push finishes inside it and behaves as before: its checkpoints are on the remote when `git push` returns. A queue that does not fit — a backlog, a slow uplink — is handed to one detached worker per repository (`entire __checkpoint_upload`, `strategy/checkpoint_upload_worker.go`), and the hook prints `Uploading the remaining N checkpoint ref(s) in the background.` and returns. Only a **budget** stop is handed off: an unreachable, refusing, or unauthenticated remote would fail the worker the same way, so those stay queued for the next push as before. + +- **One flush at a time.** Every flush takes the upload lock in the git common dir (`checkpoint.UploadCoordinator`). The worker holds it for each pass. A pre-push that finds it held records a request instead of pushing the same refs alongside it (a foreground-only pre-push waits up to 30s first); the migration push says it is waiting and waits up to 10 minutes. Whoever releases the lock starts a worker if a request is waiting (`releaseUploadLock`), so a request left while a push or migration held it is not stranded until the next push. +- **The worker acts on the hook's decisions.** The hook resolves the OPF decision before taking the lock — it may prompt, and no flush should wait on a user. The request (`checkpoint.UploadRequest`, `entire-checkpoint-upload-state.json`) carries the remote, that decision as run / skip / unset (unset falls back to the non-interactive rules, never to skip), and the pending checkpoint-sync capture, which the worker commits only after its own delivery lands. Merged requests keep a "run". The worker uploads nothing if settings cannot be loaded or redaction cannot be configured (the OPF gate would otherwise read "OPF off"), and nothing for a "run" request when OPF is off in its own process. +- **Nothing it would print is lost.** The worker's stdio is the null device (`execx.SpawnDetached`). Its last run is recorded — pushed, still queued, a one-line reason it stopped short, and any announcement such as a captured sync remote — and the next push prints it once; `entire status` shows "uploading in the background" while it runs and the last stop reason while refs stay queued. +- **Bounded.** Each pass flushes under a 10-minute budget, the run under 30 minutes; a pass that leaves refs queued ends the run (the next push retries), and further passes run only for requests that arrived meanwhile. A run record older than the run deadline is stale, not running. +- **Uploads stay inline** when `ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND=1` (set by `entire trail create`, which reports on the hook's delivery, and by the test harnesses), in CI (`CI`, `GITHUB_ACTIONS`, … — the checkout is discarded with the job), and with `strategy_options.background_checkpoint_upload: false`. They then use the full `checkpointFlushBudget`. + +Accepted limits: the queue is shared by every worktree, so a worker resolves settings from the worktree that spawned it — as a foreground flush from any worktree already does. The worker runs in that worktree, which on Windows keeps `git worktree remove` from deleting it until the run ends. A killed worker reads as running in `entire status` until its record goes stale. A worker started in an environment torn down seconds after the push loses its remaining refs with the disk, which is why the inline budget covers ordinary pushes. + An early abort also rotates that flush's failures to the back of the queue (`PushQueue.Rotate`). The queue drains in first-seen order, so without it a prefix that always fails — five checkpoints blocked by a ruleset, say — is retried in the same order on every push and the healthy refs behind it are never attempted at all. Rotation is skipped after an interruption, where the failures say nothing about the refs. Two diagnostics make a wholesale failure readable. Each failed chunk's error is logged before the fallback runs: the per-ref retries re-derive a *rejection* reason, but a transport failure — an unreachable remote, a stalled connection — matches none of them, so without that line the cause reached neither the terminal nor `.entire/logs`. And `remote.Fetch` now annotates its error with git's own output — redacted and capped by the same `errWithGitOutput` helper the push path uses — instead of the strategy layer substituting that output for the error, which reported a contentless `fetch failed:` per ref whenever git was killed before writing anything. @@ -269,6 +281,8 @@ When checkpoints *are* actively migrated from the branch into refs (a path that | `checkpoint/refs_naming.go` | `RefName` / `ParseRef`, `CheckpointRefPrefix` | | `checkpoint/refs_store.go` | `gitRefsStore` — per-checkpoint write/read, on-demand fetch | | `checkpoint/pushqueue.go` | Flock JSONL push-discovery queue | +| `checkpoint/upload_state.go` | Background upload lock, hand-off request, last-run record | +| `strategy/checkpoint_upload_worker.go` | Hybrid hand-off decision and the detached upload worker | | `checkpoint/routing_store.go` | `kindRoutingStore` — id-kind read + backfill-write routing across both backends | | `checkpoint/id/id.go` | `ShardFor`, `Kind`/`KindOf`, ID generation | | `settings/checkpoints.go` | `checkpoints` block parsing + env override | diff --git a/docs/development/checkpoint-implementation.md b/docs/development/checkpoint-implementation.md index 66af2ef7dd..c04c98a809 100644 --- a/docs/development/checkpoint-implementation.md +++ b/docs/development/checkpoint-implementation.md @@ -67,6 +67,7 @@ The manual-commit strategy (`manual_commit*.go`) does not modify the active bran - **Orphaned session state cleanup** (`isOrphanedSessionState`) - `listAllSessionStates` deletes a state only when it is ended (`State.IsEnded`: ENDED phase or `EndedAt` stamped), never condensed (no `LastCheckpointID`), and has no pending work (`!HasPendingWork()`). Legacy empty-phase records normalize to IDLE when listed, so they are no longer deleted here (they previously were) and instead remain until `StaleSessionThreshold` expiry. An IDLE never-condensed state is either a live session between turns — a read-only turn, or the turn right after a linked commit — or belongs to the exited-owner finalizer when `OwnerExited` is true. The store is listed by every worktree's hooks, so deleting either shape re-initialised live sessions from zero or erased the ownership evidence before `finalizeExitedSessions` could end the session cleanly. IDLE states therefore remain for lifecycle finalization or `StaleSessionThreshold` expiry. - PrePush hook can push `entire/checkpoints/v1` branch alongside user pushes - **Git-refs push failures stay fail-soft**: confirmed remote policy/hook rejections skip fetch/replay so repeated blocked pushes do not rewrite local commits. Other failures still attempt recovery; failed fetch/replay preserves the original push cause, not a presumed divergence. Confirmed remote rejections print one bounded example per flush with Git's line breaks intact (log errors remain single-line), without additional secret redaction of the remote's diagnostic output; unsuccessful refs stay queued. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). +- **Git-refs uploads past the inline budget continue in the background**: pre-push uploads for `checkpointInlineUploadBudget`, then hands a budget-stopped remainder to one detached `__checkpoint_upload` worker per repository. The hook resolves the OPF decision and pending sync capture and passes them in the request; the worker never re-resolves them, runs `EnsureRedactionConfigured` before uploading, and commits the capture only after its delivery lands. All flushes serialise on the upload lock. Foreground-only: `ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND=1` (`trail create`, test harnesses), CI, `strategy_options.background_checkpoint_upload: false`. See [ref-backend background upload](../architecture/ref-checkpoint-backend.md#background-upload-hybrid). - **The git-refs pre-push flush is bounded end to end**: the batch pushes `checkpointRefPushChunkSize` refs per round-trip, dequeuing each chunk as it lands, and in the pre-push path shares one `checkpointFlushBudget` deadline with the per-ref fallback (the explicit migration push leaves the batch unbounded). The chunk size adapts to the link (up to 200 refs; quartered when the budget cuts a chunk, doubled back after a clean batch), and a budget cut ends the flush without running the fallback on the spent budget. The batch stops after `maxConsecutiveChunkPushFailures` failed chunks in a row, or after the first chunk that fails to connect at all (`remote.UnreachableRemoteLine`; skips the fallback and prints that one redacted line), and the fallback after `maxConsecutiveRefPushFailures` failed refs — each always after at least one attempt, with skipped refs left queued for the next push. Each failed chunk's error is logged before the fallback runs so a transport failure, which `checkpointRefRejectionReason` deliberately does not match, still names its cause; and `remote.Fetch` annotates its error with git's redacted output via `errWithGitOutput` — deliberately not a `PushError`, which is how a refused push is recognised — rather than the caller substituting that output for the error. See [ref-backend pre-push flow](../architecture/ref-checkpoint-backend.md#pre-push-flow). - **Single checkpoint sync remote** - checkpoint data syncs only to the elected checkpoint sync remote (`strategy_options.checkpoint_push_remote` setting → captured election → `origin` → sole remote → first remote in `.git/config` order; fail-closed only when the setting names a missing remote — the captured tier is fail-soft), on both the git-branch and git-refs backends. **Capture**: during pre-push, a push whose target agrees with the branch's declared push destination (`branch..pushRemote` → `remote.pushDefault` → `branch..remote`) and differs from the current election persists that remote as the captured election (git common dir, `entire-checkpoint-sync-remotes.json`), announces it on stderr, and the same push carries the checkpoints — see `strategy/checkpoint_sync_capture.go`. Declaration alone is deliberately **not** a tier: electing from tracking config at rest silently drops checkpoint sync on every push to a different remote (`git push HEAD`, a `git clone -o base` pushing checkpoints to a separately added `origin`, any repo with `remote.pushDefault` — the `74e239a9` regression), and a bare push to an undeclared remote never elects (the pre-single-remote transcript leak). Phase-1 capture rules: at most one captured remote, first still-configured capture sticks (no per-push ping-pong in mixed-habit repos; a captured remote that was renamed or removed no longer vetoes the next capture), explicit `checkpoint_push_remote` outranks and disables capture. Pushes to any other remote or to a raw URL never carry checkpoint data; the dedicated `checkpoint_remote` URL mode is exempt. `entire status` shows the sync destination, its source (`observed` renders as "follows your branch's push destination"), and an unpushed-checkpoint count. - **A refused `checkpoint_remote` says so where it will be seen** - `checkpointRemoteIsInherited` refuses a committed `checkpoint_remote` whose owner does not match every remote identifying this repo, and used to say so only in a WARN. The visible symptom — checkpoints arriving in the code repository — is a working setup, just not the one that was asked for, so the rejection now reaches the user in `entire status` (text and `--json`, with `checkpoint_remote_ignored_verdict`) and `entire enable`, which take one verdict from `ignoredCheckpointRemote` and print the same sentence — "Checkpoints sync to X, not to the configured checkpoint_remote Y: reason" — plus the claim command built by `remote.ClaimCheckpointRemoteCommand`, for a disproved owner (a fork; rendered as information) and an unprovable one (a warning) alike. With `push_sessions` disabled both commands judge the read path instead, and also report a store the fetch side declined for a non-ownership reason. Pre-push stderr (`warnIgnoredCheckpointRemote`) is narrower: it warns only for an **unprovable** owner, because a disproved one is the ordinary fork contributor, for whom checkpoints landing in their fork is the correct outcome and a warning on every push would be noise they could silence only by claiming someone else's store. The repo is printed through `tuiutil.SanitizeDisplayText`, because it comes from the committed settings file. A local settings file present in the index or HEAD is never accepted as consent: `entire enable --local --checkpoint-remote` refuses to write it and says to untrack it, and the enable claim prompt is not offered. The enable claim prompt respects `--yes` and `--agent`, and any successful claim precedes destination resolution for the final report. It blocks nothing: checkpoints keep flowing to the elected remote, because refusing to push them turns a misconfiguration into lost work. The pre-push warning fires on **every push that delivered checkpoints** while the condition holds — a one-shot notice is seen by whoever set the repo up and not by whoever hits the problem — and is emitted after delivery, so a push carrying no checkpoints says nothing. Two gates guard it and neither is redundant: `checkpointRemoteConfigured` is the free half (an empty `checkpointURL` means "none configured" as often as "configured and refused"), and `hasCheckpointURL` is authoritative, because `PushURL` decided where this push goes while `InheritedCheckpointRemote` re-runs the vote over an identity set that can diverge from it. The claim command is offered **only for providers `parseCheckpointRemoteFlag` accepts** (github, and gitlab including nested groups): `GetCheckpointRemote` validates no provider at all, so naming any other would print a command that fails when pasted. diff --git a/e2e/controlplane/main_test.go b/e2e/controlplane/main_test.go index 79d4724275..158a293ebb 100644 --- a/e2e/controlplane/main_test.go +++ b/e2e/controlplane/main_test.go @@ -66,6 +66,7 @@ func TestMain(m *testing.M) { os.Setenv("ENTIRE_TOKEN_STORE_PATH", filepath.Join(stateDir, "tokens.json")) os.Setenv("ENTIRE_CONFIG_DIR", filepath.Join(stateDir, "entire-config")) os.Setenv("XDG_CACHE_HOME", filepath.Join(stateDir, "entire-cache")) + os.Setenv("ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND", "1") // See e2e/tests/main_test.go. // ENTIRE_TOKEN's presence alone switches the CLI to token mode, and a set // ENTIRE_AUTH_BASE_URL (retired) fails every command. for _, name := range []string{"ENTIRE_TOKEN", "ENTIRE_CONTEXT", "ENTIRE_AUTH_BASE_URL"} { diff --git a/e2e/tests/main_test.go b/e2e/tests/main_test.go index ae3fe7e169..ba9c84330a 100644 --- a/e2e/tests/main_test.go +++ b/e2e/tests/main_test.go @@ -43,6 +43,9 @@ func TestMain(m *testing.M) { // fallback cannot protect it). os.Setenv("ENTIRE_CONFIG_DIR", filepath.Join(runDir, "entire-config")) os.Setenv("XDG_CACHE_HOME", filepath.Join(runDir, "entire-cache")) + // Upload checkpoints inside the pre-push hook: a detached background + // upload would outlive the test and race its checks of the remote. + os.Setenv("ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND", "1") // And clear the agents' caller-session variables. E2E is usually run from // inside an agent, which publishes its session ID into this process's From f8aca603ab522938f3f1357740a98c128b58f4b4 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 18:28:09 -0700 Subject: [PATCH 10/16] feat(checkpoint): move past a stalled chunk in the background upload Inside a worker pass each chunk push is now bounded on its own (checkpointUploadChunkTimeout, 75s). A chunk that outlasts it is recorded as stalled, rotated to the back of the queue, halves the next chunk size, and the pass carries on, so one stalled upload no longer holds every chunk behind it for the whole delivery budget. Stalls do not count toward the consecutive-failure stop. Chunks still go out one at a time: parallel pushes would split a slow uplink and multiply auth failures, for a speedup not yet measured against a real forge. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4F4CQ950VWSXJDJDERB1ESD --- .../cli/strategy/checkpoint_upload_worker.go | 8 +++- .../strategy/checkpoint_upload_worker_test.go | 37 +++++++++++++++++++ cmd/entire/cli/strategy/manual_commit_push.go | 19 +++++++++- cmd/entire/cli/strategy/push_common.go | 25 ++++++++++++- docs/architecture/ref-checkpoint-backend.md | 1 + 5 files changed, 85 insertions(+), 5 deletions(-) diff --git a/cmd/entire/cli/strategy/checkpoint_upload_worker.go b/cmd/entire/cli/strategy/checkpoint_upload_worker.go index eed74a04ef..f41ec5c5aa 100644 --- a/cmd/entire/cli/strategy/checkpoint_upload_worker.go +++ b/cmd/entire/cli/strategy/checkpoint_upload_worker.go @@ -74,6 +74,11 @@ var ciEnvVars = []string{"CI", "GITHUB_ACTIONS", "GITLAB_CI", "BUILDKITE", "JENK // otherwise upload inline, as before. var backgroundUploadInTests = false +// checkpointUploadChunkTimeout bounds one chunk push inside a worker pass, so a +// stalled upload is moved to the back of the queue instead of holding every +// chunk behind it for the whole delivery budget. Var for tests. +var checkpointUploadChunkTimeout = max(time.Minute, checkpointUploadDeliveryBudget/8) + // checkpointUploadSpawn is the process-spawn seam, swapped in tests: // execx.SpawnDetached is a no-op under `go test`. var checkpointUploadSpawn = func(worktreeRoot string) { @@ -330,7 +335,8 @@ func runCheckpointUploadPass(ctx context.Context, repo *git.Repository, coord *c deliverCtx, cancel := context.WithTimeout(ctx, checkpointUploadDeliveryBudget) defer cancel() - res, err := flushCheckpointRefsQueue(deliverCtx, repo, ps, flushOptions{budget: checkpointUploadDeliveryBudget}) + res, err := flushCheckpointRefsQueue(deliverCtx, repo, ps, flushOptions{ + budget: checkpointUploadDeliveryBudget, chunkTimeout: checkpointUploadChunkTimeout}) if err != nil { logging.Warn(ctx, "background checkpoint upload: refs left queued", slog.String("error", err.Error())) } diff --git a/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go b/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go index d0a7ae8094..fe59621156 100644 --- a/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go +++ b/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go @@ -333,3 +333,40 @@ func TestRunCheckpointUploadWorker_WithholdsWhenRequiredOPFIsOff(t *testing.T) { require.NotNil(t, st.Last) assert.Contains(t, st.Last.Error, "privacy filter") } + +// TestRunCheckpointUploadWorker_StalledChunkDoesNotHoldTheRest: a chunk that +// outlasts the per-chunk timeout moves to the back of the queue and the pass +// carries on, instead of one stalled upload holding every chunk behind it. +func TestRunCheckpointUploadWorker_StalledChunkDoesNotHoldTheRest(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 6) + prepareGitRefsPrePush(t, workDir, bareDir) + hook := "#!/bin/sh\ngrep -q '" + refs[0].String() + "' && sleep 30\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) + restoreTimeout := checkpointUploadChunkTimeout + checkpointUploadChunkTimeout = 2 * time.Second + t.Cleanup(func() { checkpointUploadChunkTimeout = restoreTimeout }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + queue := enqueueRefs(t, repo, refs) + coord := uploadCoordinator(t, repo) + require.NoError(t, coord.Request(checkpoint.UploadRequest{Remote: "origin"})) + + start := time.Now() + RunCheckpointUploadWorker(t.Context()) + assert.Less(t, time.Since(start), 20*time.Second, "the stalled chunk is cut, not waited out") + + for _, ref := range refs[2:] { + assert.Equal(t, refHashOf(t, repo, ref), remoteRefHash(t, bareDir, ref), "chunks behind the stall still land") + } + remaining, err := queue.Drain() + require.NoError(t, err) + assert.ElementsMatch(t, refs[:2], remaining, "the stalled chunk stays queued") + assert.Equal(t, 1, queue.ChunkSizeHint(checkpointRefPushChunkSize), "a stall halves the next chunk size") + st, err := coord.State() + require.NoError(t, err) + require.NotNil(t, st.Last) + assert.Contains(t, st.Last.Error, "stalled") +} diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index ff659d7723..b4e5f965bc 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -648,7 +648,7 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push // Fast path: push the refs a chunk per round-trip (fast-forward-only). chunkSize := queue.ChunkSizeHint(checkpointRefPushChunkSize) - batch := pushRefChunks(batchCtx, dest.target, existing, chunkSize, func(landed []plumbing.ReferenceName) { + batch := pushRefChunks(batchCtx, dest.target, existing, chunkSize, opts.chunkTimeout, func(landed []plumbing.ReferenceName) { if removeErr := queue.Remove(landed); removeErr != nil { logging.Warn(ctx, "git-refs push: clear pushed refs from queue failed", slog.String("error", removeErr.Error())) @@ -660,6 +660,18 @@ func flushCheckpointRefsQueue(ctx context.Context, repo *git.Repository, ps push defer cancelFlush() budgetCut := budgetExhausted() adaptChunkSize(ctx, queue, chunkSize, batch, budgetCut) + // Stalled chunks stay queued like refs never tried, behind the rest of the + // queue so the next pass starts with refs that may move. + if len(batch.stalled) > 0 { + if err := queue.Rotate(batch.stalled); err != nil { + logging.Warn(ctx, "git-refs push: rotate stalled refs to queue back failed", + slog.String("error", err.Error())) + } + batch.untried = append(batch.untried, batch.stalled...) + if batch.stopReason == "" { + batch.stopReason = fmt.Sprintf("%d checkpoint ref(s) stalled", len(batch.stalled)) + } + } if res, done, err := settleBatchOnly(ctx, flushCtx, queue, batch, len(existing), dest, opts, stop, budgetCut); done { return res, err } @@ -816,6 +828,9 @@ type flushOptions struct { // set; 0 means checkpointFlushBudget. budget time.Duration boundBatch bool + // chunkTimeout bounds each chunk push on its own (see pushRefChunks); 0 + // leaves chunks bounded only by budget. + chunkTimeout time.Duration // handoff marks a flush whose budget stop the caller continues in the // background: that stop is reported by the caller, not as "stay queued". handoff bool @@ -915,7 +930,7 @@ func stopOnBudgetCut(ctx, flushCtx context.Context, queue *checkpoint.PushQueue, func adaptChunkSize(ctx context.Context, queue *checkpoint.PushQueue, size int, batch chunkPushResult, budgetCut bool) { next := size switch { - case budgetCut && batch.landed == 0 && len(batch.failed) > 0: + case batch.landed == 0 && (budgetCut && len(batch.failed) > 0 || len(batch.stalled) > 0): next = max(1, size/4) case len(batch.failed) == 0 && len(batch.untried) == 0: next = min(checkpointRefPushChunkSize, size*2) diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 54a1a5dd40..1b23158c2f 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -74,6 +74,7 @@ type chunkPushResult struct { landed int // refs in chunks that pushed cleanly failed []plumbing.ReferenceName // refs of chunks that failed, for the per-ref fallback untried []plumbing.ReferenceName // refs never attempted because the phase stopped + stalled []plumbing.ReferenceName // refs of chunks cut by the per-chunk timeout stopReason string // why untried is non-empty firstErr error sshAuthFailed bool @@ -91,7 +92,14 @@ type chunkPushResult struct { // row does, and so does an unreachable remote before any chunk landed: per-ref // retries cannot reach it either, and each would wait out its own connect // timeout. The first chunk is always attempted, matching the per-ref fallback. -func pushRefChunks(ctx context.Context, target string, refs []plumbing.ReferenceName, size int, onLanded func([]plumbing.ReferenceName)) chunkPushResult { +// +// chunkTimeout, when positive, bounds each chunk on its own: a chunk that +// outlasts it is recorded as stalled and the phase moves on to the next, so one +// stalled upload does not hold every chunk behind it. A stall is slowness, not +// a refusal, so it does not count toward the consecutive-failure stop. +func pushRefChunks(ctx context.Context, target string, refs []plumbing.ReferenceName, size int, + chunkTimeout time.Duration, onLanded func([]plumbing.ReferenceName), +) chunkPushResult { var res chunkPushResult consecutive := 0 for start := 0; start < len(refs); start += size { @@ -105,7 +113,20 @@ func pushRefChunks(ctx context.Context, target string, refs []plumbing.Reference } } chunk := refs[start:min(start+size, len(refs))] - err := batchPushRefs(ctx, target, chunk) + chunkCtx, cancel := ctx, context.CancelFunc(func() {}) + if chunkTimeout > 0 { + chunkCtx, cancel = context.WithTimeout(ctx, chunkTimeout) + } + err := batchPushRefs(chunkCtx, target, chunk) + stalled := err != nil && chunkTimeout > 0 && + errors.Is(chunkCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil + cancel() + if stalled { + res.stalled = append(res.stalled, chunk...) + logging.Warn(ctx, "git-refs push: checkpoint ref chunk stalled; moving on", + slog.Int("refs", len(chunk)), slog.Duration("timeout", chunkTimeout)) + continue + } if err == nil { consecutive = 0 res.landed += len(chunk) diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index 927127aae4..9c1c619f15 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -131,6 +131,7 @@ The pre-push hook uploads inline for only `checkpointInlineUploadBudget` (5s). A - **One flush at a time.** Every flush takes the upload lock in the git common dir (`checkpoint.UploadCoordinator`). The worker holds it for each pass. A pre-push that finds it held records a request instead of pushing the same refs alongside it (a foreground-only pre-push waits up to 30s first); the migration push says it is waiting and waits up to 10 minutes. Whoever releases the lock starts a worker if a request is waiting (`releaseUploadLock`), so a request left while a push or migration held it is not stranded until the next push. - **The worker acts on the hook's decisions.** The hook resolves the OPF decision before taking the lock — it may prompt, and no flush should wait on a user. The request (`checkpoint.UploadRequest`, `entire-checkpoint-upload-state.json`) carries the remote, that decision as run / skip / unset (unset falls back to the non-interactive rules, never to skip), and the pending checkpoint-sync capture, which the worker commits only after its own delivery lands. Merged requests keep a "run". The worker uploads nothing if settings cannot be loaded or redaction cannot be configured (the OPF gate would otherwise read "OPF off"), and nothing for a "run" request when OPF is off in its own process. - **Nothing it would print is lost.** The worker's stdio is the null device (`execx.SpawnDetached`). Its last run is recorded — pushed, still queued, a one-line reason it stopped short, and any announcement such as a captured sync remote — and the next push prints it once; `entire status` shows "uploading in the background" while it runs and the last stop reason while refs stay queued. +- **A stalled chunk does not hold the rest.** Inside a pass each chunk push is also bounded on its own (`checkpointUploadChunkTimeout`, 75s): a chunk that outlasts it moves to the back of the queue, halves the next chunk size, and the pass carries on. Chunks still go out one at a time — parallel pushes would split a slow uplink four ways and multiply auth failures, for a speedup not yet measured against a real forge. - **Bounded.** Each pass flushes under a 10-minute budget, the run under 30 minutes; a pass that leaves refs queued ends the run (the next push retries), and further passes run only for requests that arrived meanwhile. A run record older than the run deadline is stale, not running. - **Uploads stay inline** when `ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND=1` (set by `entire trail create`, which reports on the hook's delivery, and by the test harnesses), in CI (`CI`, `GITHUB_ACTIONS`, … — the checkout is discarded with the job), and with `strategy_options.background_checkpoint_upload: false`. They then use the full `checkpointFlushBudget`. From 7c957b647f802b4fb26e8ee3a7a6da1eef32c7de Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 19:47:53 -0700 Subject: [PATCH 11/16] fix(execx): keep a detached child from holding git's pipes open os/exec passes a child only its stdio and ExtraFiles, but it does not close descriptors the process inherited without close-on-exec. A git pre-push hook inherits git's pipes that way, including the write end of a remote helper's stdin, so the detached checkpoint upload worker kept git-remote-entire from seeing EOF and the user's `git push` to an entire:// remote could not exit until the worker finished (measured 19-82s against Entire; 7s after). SpawnDetached now marks every inherited descriptor above stderr close-on-exec before starting the child. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4F8YP3304B1N3ZT9BHT14DT --- cmd/entire/cli/execx/execx_unix.go | 28 +++++++++++++++++++ cmd/entire/cli/execx/execx_unix_test.go | 30 +++++++++++++++++++++ cmd/entire/cli/execx/execx_windows.go | 4 +++ cmd/entire/cli/execx/spawn_detached.go | 3 +++ docs/architecture/ref-checkpoint-backend.md | 1 + 5 files changed, 66 insertions(+) diff --git a/cmd/entire/cli/execx/execx_unix.go b/cmd/entire/cli/execx/execx_unix.go index fd611b280d..b80e93ab17 100644 --- a/cmd/entire/cli/execx/execx_unix.go +++ b/cmd/entire/cli/execx/execx_unix.go @@ -4,7 +4,10 @@ package execx import ( "fmt" + "os" "os/exec" + "runtime" + "strconv" "syscall" ) @@ -36,3 +39,28 @@ func killProcessGroupOnCancel(cmd *exec.Cmd) { return nil } } + +// markInheritedFDsCloseOnExec sets close-on-exec on every descriptor above +// stderr. os/exec passes a child only its stdio and ExtraFiles, but it does not +// close descriptors this process inherited without close-on-exec: those pass on +// to every child. A git hook inherits git's pipes that way — the write end of a +// remote helper's stdin among them — and a detached child holding one keeps +// the helper from seeing EOF, so the user's `git push` cannot finish until the +// child exits. Descriptors Go opened itself are already close-on-exec. +func markInheritedFDsCloseOnExec() { + dir := "/dev/fd" + if runtime.GOOS == "linux" { + dir = "/proc/self/fd" + } + entries, err := os.ReadDir(dir) + if err != nil { + return + } + for _, e := range entries { + fd, err := strconv.Atoi(e.Name()) + if err != nil || fd <= 2 { + continue + } + syscall.CloseOnExec(fd) + } +} diff --git a/cmd/entire/cli/execx/execx_unix_test.go b/cmd/entire/cli/execx/execx_unix_test.go index 781f3e33cb..d0d6301f33 100644 --- a/cmd/entire/cli/execx/execx_unix_test.go +++ b/cmd/entire/cli/execx/execx_unix_test.go @@ -7,7 +7,9 @@ import ( "context" "io" "os/exec" + "strconv" "strings" + "syscall" "testing" "time" ) @@ -110,3 +112,31 @@ func TestKillProcessGroupOnCancel_TerminatesLeader(t *testing.T) { t.Fatal("Wait did not return after Cancel; the group-kill handler may have regressed") } } + +// TestMarkInheritedFDsCloseOnExec: a descriptor this process holds without +// close-on-exec — as a git hook holds git's pipes — must not reach a child +// started afterwards, or a detached child keeps the pipe open for its lifetime. +func TestMarkInheritedFDsCloseOnExec(t *testing.T) { + var fds [2]int + if err := syscall.Pipe(fds[:]); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { syscall.Close(fds[0]); syscall.Close(fds[1]) }) + leaked := strconv.Itoa(fds[1]) + childFDs := func() string { + out, err := exec.CommandContext(t.Context(), "sh", "-c", "ls /dev/fd").Output() + if err != nil { + t.Fatal(err) + } + return " " + strings.Join(strings.Fields(string(out)), " ") + " " + } + if !strings.Contains(childFDs(), " "+leaked+" ") { + t.Skip("precondition: this platform already keeps inherited descriptors from children") + } + + markInheritedFDsCloseOnExec() + + if got := childFDs(); strings.Contains(got, " "+leaked+" ") { + t.Errorf("child still inherited fd %s: %s", leaked, got) + } +} diff --git a/cmd/entire/cli/execx/execx_windows.go b/cmd/entire/cli/execx/execx_windows.go index fb0f3caa10..d086b4c795 100644 --- a/cmd/entire/cli/execx/execx_windows.go +++ b/cmd/entire/cli/execx/execx_windows.go @@ -22,3 +22,7 @@ func detachFromTTY(cmd *exec.Cmd) { // killProcessGroupOnCancel is a no-op on Windows: reliable tree-kill needs a Job // Object. The WaitDelay backstop still bounds the wait on a hung subprocess. func killProcessGroupOnCancel(_ *exec.Cmd) {} + +// markInheritedFDsCloseOnExec is a no-op on Windows: os/exec starts children +// with an explicit handle list, so no other handle is inherited. +func markInheritedFDsCloseOnExec() {} diff --git a/cmd/entire/cli/execx/spawn_detached.go b/cmd/entire/cli/execx/spawn_detached.go index 846e225055..6323bd5fc3 100644 --- a/cmd/entire/cli/execx/spawn_detached.go +++ b/cmd/entire/cli/execx/spawn_detached.go @@ -28,6 +28,9 @@ func SpawnDetached(dir string, args ...string) { } cmd := detachedCommand(executable, dir, args...) + // The child outlives this process; nothing else this process inherited may + // outlive it with the child (see markInheritedFDsCloseOnExec). + markInheritedFDsCloseOnExec() if err := cmd.Start(); err != nil { return } diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index 9c1c619f15..e3707d34aa 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -130,6 +130,7 @@ The pre-push hook uploads inline for only `checkpointInlineUploadBudget` (5s). A - **One flush at a time.** Every flush takes the upload lock in the git common dir (`checkpoint.UploadCoordinator`). The worker holds it for each pass. A pre-push that finds it held records a request instead of pushing the same refs alongside it (a foreground-only pre-push waits up to 30s first); the migration push says it is waiting and waits up to 10 minutes. Whoever releases the lock starts a worker if a request is waiting (`releaseUploadLock`), so a request left while a push or migration held it is not stranded until the next push. - **The worker acts on the hook's decisions.** The hook resolves the OPF decision before taking the lock — it may prompt, and no flush should wait on a user. The request (`checkpoint.UploadRequest`, `entire-checkpoint-upload-state.json`) carries the remote, that decision as run / skip / unset (unset falls back to the non-interactive rules, never to skip), and the pending checkpoint-sync capture, which the worker commits only after its own delivery lands. Merged requests keep a "run". The worker uploads nothing if settings cannot be loaded or redaction cannot be configured (the OPF gate would otherwise read "OPF off"), and nothing for a "run" request when OPF is off in its own process. +- **It never holds the user's push open.** `execx.SpawnDetached` marks every descriptor the hook inherited from git close-on-exec before starting the worker. Otherwise the worker inherits the write end of a remote helper's stdin (`git-remote-entire` for `entire://` remotes), the helper never sees EOF, and `git push` cannot exit until the worker does — measured as a 19–82s push against Entire before the fix, 7s after. - **Nothing it would print is lost.** The worker's stdio is the null device (`execx.SpawnDetached`). Its last run is recorded — pushed, still queued, a one-line reason it stopped short, and any announcement such as a captured sync remote — and the next push prints it once; `entire status` shows "uploading in the background" while it runs and the last stop reason while refs stay queued. - **A stalled chunk does not hold the rest.** Inside a pass each chunk push is also bounded on its own (`checkpointUploadChunkTimeout`, 75s): a chunk that outlasts it moves to the back of the queue, halves the next chunk size, and the pass carries on. Chunks still go out one at a time — parallel pushes would split a slow uplink four ways and multiply auth failures, for a speedup not yet measured against a real forge. - **Bounded.** Each pass flushes under a 10-minute budget, the run under 30 minutes; a pass that leaves refs queued ends the run (the next push retries), and further passes run only for requests that arrived meanwhile. A run record older than the run deadline is stale, not running. From 552399c14b48647a3bcece2397e7f2767dc8addf Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Thu, 8 Oct 2026 20:20:12 -0700 Subject: [PATCH 12/16] fix(checkpoint): background upload review fixes - A chunk down to one ref is no longer cut by the worker's per-chunk timeout: a ref slower than the timeout was re-sent on every pass and never landed. It gets the rest of the pass budget. - Inline-only uploads (CI, trail create, the setting) never start a background process: a push that finds the lock held leaves its refs queued, and releasing the lock spawns a worker only where background upload is allowed. - entire status no longer reports a killed worker as running: a run record without a finish time counts as running only while the upload lock is held. - Inherited-descriptor marking falls back to scanning up to RLIMIT_NOFILE where /dev/fd or /proc/self/fd is unavailable. - A stall shrinks the chunk size only when nothing landed, matching the budget-cut rule. - Wording no longer assumes the lock holder is a background worker, and a failed spawn is not reported as a hand-off. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4FASWG4K5H5EQS6XF365500 --- cmd/entire/cli/execx/execx_unix.go | 15 ++++ cmd/entire/cli/execx/execx_unix_test.go | 2 + .../cli/strategy/checkpoint_upload_worker.go | 24 ++++-- .../strategy/checkpoint_upload_worker_test.go | 81 ++++++++++++++++++- cmd/entire/cli/strategy/manual_commit_push.go | 8 +- cmd/entire/cli/strategy/push_common.go | 7 +- 6 files changed, 126 insertions(+), 11 deletions(-) diff --git a/cmd/entire/cli/execx/execx_unix.go b/cmd/entire/cli/execx/execx_unix.go index b80e93ab17..746d019de7 100644 --- a/cmd/entire/cli/execx/execx_unix.go +++ b/cmd/entire/cli/execx/execx_unix.go @@ -47,6 +47,10 @@ func killProcessGroupOnCancel(cmd *exec.Cmd) { // remote helper's stdin among them — and a detached child holding one keeps // the helper from seeing EOF, so the user's `git push` cannot finish until the // child exits. Descriptors Go opened itself are already close-on-exec. +// +// It runs in the hook process, so later children of the hook lose those +// descriptors too; the one legitimate use, a user's GIT_TRACE=, makes git +// warn and carry on. func markInheritedFDsCloseOnExec() { dir := "/dev/fd" if runtime.GOOS == "linux" { @@ -54,6 +58,17 @@ func markInheritedFDsCloseOnExec() { } entries, err := os.ReadDir(dir) if err != nil { + // No fd directory (no /proc in a container, BSD without fdescfs): + // mark every possible descriptor instead. CloseOnExec on a closed + // one is a harmless EBADF. + var lim syscall.Rlimit + maxFD := uint64(4096) + if syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim) == nil && lim.Cur < maxFD { + maxFD = lim.Cur + } + for fd := 3; uint64(fd) < maxFD; fd++ { + syscall.CloseOnExec(fd) + } return } for _, e := range entries { diff --git a/cmd/entire/cli/execx/execx_unix_test.go b/cmd/entire/cli/execx/execx_unix_test.go index d0d6301f33..2e0ec7fcee 100644 --- a/cmd/entire/cli/execx/execx_unix_test.go +++ b/cmd/entire/cli/execx/execx_unix_test.go @@ -116,6 +116,8 @@ func TestKillProcessGroupOnCancel_TerminatesLeader(t *testing.T) { // TestMarkInheritedFDsCloseOnExec: a descriptor this process holds without // close-on-exec — as a git hook holds git's pipes — must not reach a child // started afterwards, or a detached child keeps the pipe open for its lifetime. +// +// Not parallel: it changes descriptor flags process-wide. func TestMarkInheritedFDsCloseOnExec(t *testing.T) { var fds [2]int if err := syscall.Pipe(fds[:]); err != nil { diff --git a/cmd/entire/cli/strategy/checkpoint_upload_worker.go b/cmd/entire/cli/strategy/checkpoint_upload_worker.go index f41ec5c5aa..f51b82682c 100644 --- a/cmd/entire/cli/strategy/checkpoint_upload_worker.go +++ b/cmd/entire/cli/strategy/checkpoint_upload_worker.go @@ -52,8 +52,9 @@ const CheckpointUploadForegroundEnv = "ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND" var checkpointInlineUploadBudget = 5 * time.Second // foregroundUploadLockWait bounds how long a foreground flush (hand-off -// disabled) waits for a running worker before leaving its refs queued. -const foregroundUploadLockWait = 30 * time.Second +// disabled) waits for a running worker before leaving its refs queued. Var for +// tests. +var foregroundUploadLockWait = 30 * time.Second const ( // checkpointUploadDeliveryBudget bounds one worker pass's flush. @@ -201,20 +202,23 @@ func handOffCheckpointUpload(ctx context.Context, coord *checkpoint.UploadCoordi slog.String("error", err.Error())) return false } - spawnCheckpointUploadWorker(ctx) + if !spawnCheckpointUploadWorker(ctx) { + return false + } logging.Info(logging.WithComponent(ctx, checkpointUploadComponent), "handed checkpoint upload to background worker", slog.String("remote", req.Remote)) return true } -func spawnCheckpointUploadWorker(ctx context.Context) { +func spawnCheckpointUploadWorker(ctx context.Context) bool { root, err := paths.WorktreeRoot(ctx) if err != nil { logging.Debug(ctx, "skipping background checkpoint upload spawn: no worktree root", slog.String("error", err.Error())) - return + return false } checkpointUploadSpawn(root) + return true } // releaseUploadLock releases a flush's hold on the upload lock and, if a push @@ -223,7 +227,7 @@ func spawnCheckpointUploadWorker(ctx context.Context) { // it spawned exited at once, since the lock was taken. func releaseUploadLock(ctx context.Context, coord *checkpoint.UploadCoordinator, release func()) { release() - if coord != nil && !inUploadWorker(ctx) && coord.HasRequest() { + if coord != nil && !inUploadWorker(ctx) && coord.HasRequest() && backgroundCheckpointUploadEnabled(ctx) { spawnCheckpointUploadWorker(ctx) } } @@ -392,6 +396,14 @@ func CheckpointUploadStatus(ctx context.Context) (running bool, lastError string return false, "" } running = st.Last.Running(time.Now(), checkpointUploadWorkerDeadline) + // A worker that was killed leaves a record that still reads as running, + // but the kernel freed its lock: a lock we can take means nobody uploads. + if running { + if release, ok, lockErr := coord.TryLockWorker(ctx); lockErr == nil && ok { + release() + running = false + } + } if running { return true, "" } diff --git a/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go b/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go index fe59621156..c20b661f8f 100644 --- a/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go +++ b/cmd/entire/cli/strategy/checkpoint_upload_worker_test.go @@ -127,7 +127,7 @@ func TestPrePush_RunningWorkerOwnsTheQueue(t *testing.T) { output := restore() assert.Equal(t, 0, countedAttempts(t, countFile), "nothing pushed alongside the running worker") - assert.Contains(t, output, "already running in the background") + assert.Contains(t, output, "will follow it in the background") remaining, err := queue.Drain() require.NoError(t, err) assert.ElementsMatch(t, refs, remaining) @@ -307,6 +307,12 @@ func TestReleaseUploadLock_StartsWorkerForWaitingRequest(t *testing.T) { require.NoError(t, err) releaseUploadLock(withinUploadWorker(t.Context()), coord, release) assert.Len(t, *spawned, 1, "the worker loops over requests itself") + + t.Setenv(CheckpointUploadForegroundEnv, "1") + release, err = coord.LockWorker(t.Context()) + require.NoError(t, err) + releaseUploadLock(t.Context(), coord, release) + assert.Len(t, *spawned, 1, "uploads kept inline never start a background process") } // TestRunCheckpointUploadWorker_WithholdsWhenRequiredOPFIsOff: the hook @@ -364,9 +370,80 @@ func TestRunCheckpointUploadWorker_StalledChunkDoesNotHoldTheRest(t *testing.T) remaining, err := queue.Drain() require.NoError(t, err) assert.ElementsMatch(t, refs[:2], remaining, "the stalled chunk stays queued") - assert.Equal(t, 1, queue.ChunkSizeHint(checkpointRefPushChunkSize), "a stall halves the next chunk size") + assert.Equal(t, 2, queue.ChunkSizeHint(checkpointRefPushChunkSize), "chunks behind the stall landed, so the size stays") st, err := coord.State() require.NoError(t, err) require.NotNil(t, st.Last) assert.Contains(t, st.Last.Error, "stalled") } + +// TestPrePush_ForegroundNeverHandsOffWhenLockHeld: an inline-only push (CI, +// trail create, the setting) that finds another upload holding the lock +// leaves its refs queued rather than starting a background process. +func TestPrePush_ForegroundNeverHandsOffWhenLockHeld(t *testing.T) { + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 2) + prepareGitRefsPrePush(t, workDir, bareDir) + spawned := enableBackgroundUpload(t, 5*time.Second) + t.Setenv(CheckpointUploadForegroundEnv, "1") + restoreWait := foregroundUploadLockWait + foregroundUploadLockWait = 200 * time.Millisecond + t.Cleanup(func() { foregroundUploadLockWait = restoreWait }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + coord := uploadCoordinator(t, repo) + release, err := coord.LockWorker(t.Context()) + require.NoError(t, err) + defer release() + + restore := captureStderr(t) + require.NoError(t, NewManualCommitStrategy().PrePushFromGitHook(t.Context(), "origin")) + output := restore() + + assert.Empty(t, *spawned) + assert.Contains(t, output, "stay queued for the next push") + st, err := coord.State() + require.NoError(t, err) + assert.Nil(t, st.Request, "no request is left for a background process") +} + +// TestRunCheckpointUploadWorker_SingleSlowRefIsNotCut: a chunk down to one ref +// gets the pass budget, not the per-chunk timeout, or a ref slower than the +// timeout would be cut and re-sent on every pass and never land. +func TestRunCheckpointUploadWorker_SingleSlowRefIsNotCut(t *testing.T) { + shrinkRefPushChunkSize(t, 1) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 1) + prepareGitRefsPrePush(t, workDir, bareDir) + hook := "#!/bin/sh\ncat >/dev/null\nsleep 3\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) + restoreTimeout := checkpointUploadChunkTimeout + checkpointUploadChunkTimeout = time.Second + t.Cleanup(func() { checkpointUploadChunkTimeout = restoreTimeout }) + + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + enqueueRefs(t, repo, refs) + require.NoError(t, uploadCoordinator(t, repo).Request(checkpoint.UploadRequest{Remote: "origin"})) + + RunCheckpointUploadWorker(t.Context()) + + assert.Equal(t, refHashOf(t, repo, refs[0]), remoteRefHash(t, bareDir, refs[0])) +} + +// TestCheckpointUploadStatus_KilledWorkerIsNotRunning: a killed worker leaves +// a run record with no finish time, but its lock was freed; status must not +// claim an upload is running. +func TestCheckpointUploadStatus_KilledWorkerIsNotRunning(t *testing.T) { + workDir, bareDir, _ := setupRepoWithNCheckpointRefs(t, 1) + prepareGitRefsPrePush(t, workDir, bareDir) + repo, err := gitrepo.OpenPath(workDir) + require.NoError(t, err) + defer repo.Close() + require.NoError(t, uploadCoordinator(t, repo).StartRun(time.Now())) + + running, _ := CheckpointUploadStatus(t.Context()) + assert.False(t, running) +} diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index b4e5f965bc..546472be02 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -450,9 +450,15 @@ func (s *ManualCommitStrategy) prePushCheckpointRefs(ctx context.Context, ps pus // a request instead of pushing the same refs alongside it. release, locked := lockQueueForFlush(ctx, coord, background) if !locked { + // Uploads kept inline (CI, trail create, the setting) never leave work + // for a background process: they stay queued for the next push. + if !background { + fmt.Fprintln(os.Stderr, "[entire] Another checkpoint upload is in progress; checkpoints stay queued for the next push.") + return nil + } if handOffCheckpointUpload(ctx, coord, checkpoint.UploadRequest{Remote: ps.remote, OPFDecision: decision, PendingCapture: pendingCapture}) { - fmt.Fprintln(os.Stderr, "[entire] A checkpoint upload is already running in the background; these checkpoints will follow it.") + fmt.Fprintln(os.Stderr, "[entire] Another checkpoint upload is in progress; these checkpoints will follow it in the background.") } return nil } diff --git a/cmd/entire/cli/strategy/push_common.go b/cmd/entire/cli/strategy/push_common.go index 1b23158c2f..5fbbbafbde 100644 --- a/cmd/entire/cli/strategy/push_common.go +++ b/cmd/entire/cli/strategy/push_common.go @@ -114,11 +114,14 @@ func pushRefChunks(ctx context.Context, target string, refs []plumbing.Reference } chunk := refs[start:min(start+size, len(refs))] chunkCtx, cancel := ctx, context.CancelFunc(func() {}) - if chunkTimeout > 0 { + // A single ref is as small as a chunk gets: timing it out would only + // re-send the same partial upload on every pass, never landing it, so + // it gets the rest of the budget instead. + if chunkTimeout > 0 && len(chunk) > 1 { chunkCtx, cancel = context.WithTimeout(ctx, chunkTimeout) } err := batchPushRefs(chunkCtx, target, chunk) - stalled := err != nil && chunkTimeout > 0 && + stalled := err != nil && chunkTimeout > 0 && len(chunk) > 1 && errors.Is(chunkCtx.Err(), context.DeadlineExceeded) && ctx.Err() == nil cancel() if stalled { From 7f7303b058e2ee2c9262be510125985569bcec24 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Fri, 9 Oct 2026 14:58:59 -0700 Subject: [PATCH 13/16] feat(checkpoint): give the pre-push hook a 10s inline upload window Timing on GitHub and Entire remotes showed 5s handed off 200 x 100KB backlogs that main delivers inline in ~11s. At 10s those land before git push returns, and larger backlogs still hand off after ~13s. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4HATE36T9TM4DKH6XTXD9WR --- cmd/entire/cli/strategy/checkpoint_upload_worker.go | 2 +- docs/architecture/ref-checkpoint-backend.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/entire/cli/strategy/checkpoint_upload_worker.go b/cmd/entire/cli/strategy/checkpoint_upload_worker.go index f51b82682c..85f4280db4 100644 --- a/cmd/entire/cli/strategy/checkpoint_upload_worker.go +++ b/cmd/entire/cli/strategy/checkpoint_upload_worker.go @@ -49,7 +49,7 @@ const CheckpointUploadForegroundEnv = "ENTIRE_CHECKPOINT_UPLOAD_FOREGROUND" // off — its checkpoints are on the remote when the push returns, which // entire.io, `entire trail create`, and short-lived environments rely on — and // short enough that a backlog no longer holds the user's push. Var for tests. -var checkpointInlineUploadBudget = 5 * time.Second +var checkpointInlineUploadBudget = 10 * time.Second // foregroundUploadLockWait bounds how long a foreground flush (hand-off // disabled) waits for a running worker before leaving its refs queued. Var for diff --git a/docs/architecture/ref-checkpoint-backend.md b/docs/architecture/ref-checkpoint-backend.md index e3707d34aa..32cca76150 100644 --- a/docs/architecture/ref-checkpoint-backend.md +++ b/docs/architecture/ref-checkpoint-backend.md @@ -126,7 +126,7 @@ The fallback is bounded. It runs under a `checkpointFlushBudget` context deadlin ### Background upload (hybrid) -The pre-push hook uploads inline for only `checkpointInlineUploadBudget` (5s). An ordinary push finishes inside it and behaves as before: its checkpoints are on the remote when `git push` returns. A queue that does not fit — a backlog, a slow uplink — is handed to one detached worker per repository (`entire __checkpoint_upload`, `strategy/checkpoint_upload_worker.go`), and the hook prints `Uploading the remaining N checkpoint ref(s) in the background.` and returns. Only a **budget** stop is handed off: an unreachable, refusing, or unauthenticated remote would fail the worker the same way, so those stay queued for the next push as before. +The pre-push hook uploads inline for only `checkpointInlineUploadBudget` (10s). An ordinary push finishes inside it and behaves as before: its checkpoints are on the remote when `git push` returns. A queue that does not fit — a backlog, a slow uplink — is handed to one detached worker per repository (`entire __checkpoint_upload`, `strategy/checkpoint_upload_worker.go`), and the hook prints `Uploading the remaining N checkpoint ref(s) in the background.` and returns. Only a **budget** stop is handed off: an unreachable, refusing, or unauthenticated remote would fail the worker the same way, so those stay queued for the next push as before. - **One flush at a time.** Every flush takes the upload lock in the git common dir (`checkpoint.UploadCoordinator`). The worker holds it for each pass. A pre-push that finds it held records a request instead of pushing the same refs alongside it (a foreground-only pre-push waits up to 30s first); the migration push says it is waiting and waits up to 10 minutes. Whoever releases the lock starts a worker if a request is waiting (`releaseUploadLock`), so a request left while a push or migration held it is not stranded until the next push. - **The worker acts on the hook's decisions.** The hook resolves the OPF decision before taking the lock — it may prompt, and no flush should wait on a user. The request (`checkpoint.UploadRequest`, `entire-checkpoint-upload-state.json`) carries the remote, that decision as run / skip / unset (unset falls back to the non-interactive rules, never to skip), and the pending checkpoint-sync capture, which the worker commits only after its own delivery lands. Merged requests keep a "run". The worker uploads nothing if settings cannot be loaded or redaction cannot be configured (the OPF gate would otherwise read "OPF off"), and nothing for a "run" request when OPF is off in its own process. From cec167147571ad510ba7a9fdb6b2e94f947eaff3 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Fri, 9 Oct 2026 15:13:52 -0700 Subject: [PATCH 14/16] fix(checkpoint): keep the chunk size after a stalled chunk A batch where some chunks landed and one stalled counted as clean and doubled the size back toward the cap. Clarify why an earlier push's OPF "run" outranks a later skip. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4HBNNCT0XDT20M7AJNVRFE6 --- cmd/entire/cli/checkpoint/upload_state.go | 5 +++-- cmd/entire/cli/strategy/manual_commit_push.go | 5 +++-- cmd/entire/cli/strategy/refs_push_bounds_test.go | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cmd/entire/cli/checkpoint/upload_state.go b/cmd/entire/cli/checkpoint/upload_state.go index 3b6def7d5d..a958085554 100644 --- a/cmd/entire/cli/checkpoint/upload_state.go +++ b/cmd/entire/cli/checkpoint/upload_state.go @@ -132,8 +132,9 @@ func (c *UploadCoordinator) LockWorker(ctx context.Context) (func(), error) { } // Request records req for the worker, merged with any request not yet taken: -// the latest push's destination wins, and so does a decision to run OPF — a -// later push that resolved nothing must not drop an earlier push's "run". +// the latest push's destination wins, and so does a decision to run OPF. The +// queue holds the earlier push's refs too, so a later push that resolved skip +// or nothing must not let them upload unscanned. func (c *UploadCoordinator) Request(req UploadRequest) error { return c.update(func(st *UploadState) { if st.Request != nil && st.Request.OPFDecision == UploadOPFRun { diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index c32039474c..13a290eb23 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -941,13 +941,14 @@ func stopOnBudgetCut(ctx, flushCtx context.Context, queue *checkpoint.PushQueue, // the queue, and each such cut costs a whole budget, hence the steep step. A // cut after chunks landed is a backlog draining as intended, not a chunk too // large, so the size is kept. It doubles back toward checkpointRefPushChunkSize -// after a batch that landed everything. +// after a batch that landed everything; a stalled chunk did not land, so it +// keeps the size too. func adaptChunkSize(ctx context.Context, queue *checkpoint.PushQueue, size int, batch chunkPushResult, budgetCut bool) { next := size switch { case batch.landed == 0 && (budgetCut && len(batch.failed) > 0 || len(batch.stalled) > 0): next = max(1, size/4) - case len(batch.failed) == 0 && len(batch.untried) == 0: + case len(batch.failed) == 0 && len(batch.untried) == 0 && len(batch.stalled) == 0: next = min(checkpointRefPushChunkSize, size*2) } if next == size { diff --git a/cmd/entire/cli/strategy/refs_push_bounds_test.go b/cmd/entire/cli/strategy/refs_push_bounds_test.go index bf76963bc7..6a3e7a2f37 100644 --- a/cmd/entire/cli/strategy/refs_push_bounds_test.go +++ b/cmd/entire/cli/strategy/refs_push_bounds_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/entireio/cli/cmd/entire/cli/checkpoint" "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" checkpointremote "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" "github.com/entireio/cli/cmd/entire/cli/entiredir" @@ -650,6 +651,20 @@ func TestFlushCheckpointRefs_CutAfterProgressKeepsChunkSize(t *testing.T) { assert.Equal(t, 2, queue.ChunkSizeHint(checkpointRefPushChunkSize), "progress was made, so the size stays") } +// TestAdaptChunkSize_StallAfterProgressKeepsSize: a batch whose other chunks +// landed but one stalled did not land everything, so the size must not grow +// back toward the cap the link just proved too slow for. +func TestAdaptChunkSize_StallAfterProgressKeepsSize(t *testing.T) { + t.Parallel() + queue := checkpoint.NewPushQueue(t.TempDir()) + require.NoError(t, queue.SetChunkSizeHint(50)) + + batch := chunkPushResult{landed: 100, stalled: []plumbing.ReferenceName{"refs/entire/checkpoints/aa/x"}} + adaptChunkSize(t.Context(), queue, 50, batch, false) + + assert.Equal(t, 50, queue.ChunkSizeHint(checkpointRefPushChunkSize)) +} + // TestFlushCheckpointRefs_SplitIsolatesRejectedRef: a forge declines a whole // push for one bad ref, failing its chunk. Halving finds that ref in a few // pushes and lands the rest, instead of retrying the chunk one ref at a time. From cc61c83717996f107b45e99ebcd4d45b5a685623 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Fri, 9 Oct 2026 15:33:17 -0700 Subject: [PATCH 15/16] fix(checkpoint): report refs left queued by an explicit push The chunked flush is fail-soft about an early stop, but PushQueuedCheckpointRefs callers treat a nil error as success, so a stop between chunks could report a partial push as complete. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4HCS7V4M0VQBTMBC891HPXG --- cmd/entire/cli/strategy/manual_commit_push.go | 9 +++++++ cmd/entire/cli/strategy/refs_push_test.go | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index 13a290eb23..28458085a9 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -587,6 +587,15 @@ func PushQueuedCheckpointRefs(ctx context.Context, repo *git.Repository, remote return 0, false, fmt.Errorf("checkpoint refs stay queued: %w", opfErr) } res, err := flushCheckpointRefsQueue(ctx, repo, ps, flushOptions{}) + if err == nil && res.remaining > 0 { + // The flush is fail-soft about an early stop, but this caller reports + // success on a nil error, so refs left behind must surface. A caller's + // cancellation stays recognisable through the wrap. + err = fmt.Errorf("%d checkpoint ref(s) stay queued: %s", res.remaining, res.failureLine(nil)) + if ctxErr := ctx.Err(); ctxErr != nil { + err = fmt.Errorf("%w: %w", err, ctxErr) + } + } return res.pushed, false, err } diff --git a/cmd/entire/cli/strategy/refs_push_test.go b/cmd/entire/cli/strategy/refs_push_test.go index d5567298c6..fdeaea5719 100644 --- a/cmd/entire/cli/strategy/refs_push_test.go +++ b/cmd/entire/cli/strategy/refs_push_test.go @@ -261,6 +261,30 @@ func TestPushQueuedCheckpointRefs_FailureLeavesRefsQueued(t *testing.T) { assert.ElementsMatch(t, refs, remaining, "failed push leaves refs queued") } +// TestPushQueuedCheckpointRefs_InterruptIsAnError: the explicit push reports +// success on a nil error, so an interrupted flush must return one that wraps +// the caller's cancellation (`doctor migrate` handles that on its own) and +// leave the refs queued. +func TestPushQueuedCheckpointRefs_InterruptIsAnError(t *testing.T) { + workDir, bareDir, refs := setupRepoWithCheckpointRefs(t) + t.Chdir(workDir) + paths.ClearWorktreeRootCache() + + repo, err := git.PlainOpen(workDir) + require.NoError(t, err) + queue := enqueueRefs(t, repo, refs) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + pushed, _, err := PushQueuedCheckpointRefs(ctx, repo, bareDir) + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 0, pushed) + + remaining, drainErr := queue.Drain() + require.NoError(t, drainErr) + assert.ElementsMatch(t, refs, remaining, "unpushed refs stay queued") +} + // remoteRefFiles lists the files in the tree a ref points at on the bare remote. func remoteRefFiles(t *testing.T, bareDir string, ref plumbing.ReferenceName) string { t.Helper() From e4be83da22ca0aefd4cb86bc0366f25f829cdcd9 Mon Sep 17 00:00:00 2001 From: Peyton Montei Date: Fri, 9 Oct 2026 18:46:04 -0700 Subject: [PATCH 16/16] test(checkpoint): cover an explicit push stopping with refs queued MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The earlier test passed without the fix: a cancelled context already failed the first chunk. Drive the real case instead — two failed chunks stop the batch, the fallback lands them, the rest stay untried. Mark that stop with a sentinel so doctor migrate does not repeat the reason the flush already printed. Co-Authored-By: Claude Opus 5.5 Entire-Checkpoint: 01M4HQT6YQBPF5N2ZQMCJ1KK6F --- cmd/entire/cli/doctor_migrate.go | 3 ++- cmd/entire/cli/strategy/manual_commit_push.go | 6 ++++- cmd/entire/cli/strategy/refs_push_test.go | 27 ++++++++++--------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/cmd/entire/cli/doctor_migrate.go b/cmd/entire/cli/doctor_migrate.go index 74c2da4f27..900ed03cc8 100644 --- a/cmd/entire/cli/doctor_migrate.go +++ b/cmd/entire/cli/doctor_migrate.go @@ -126,7 +126,8 @@ func pushMigratedRefs(ctx context.Context, out io.Writer, repo *git.Repository, pushed, pushDisabled, err := strategy.PushQueuedCheckpointRefs(ctx, repo, pushRemote) if err != nil { - if errors.Is(err, context.Canceled) { + // The flush printed why it stopped and what stays queued. + if errors.Is(err, context.Canceled) || errors.Is(err, strategy.ErrCheckpointRefsStayQueued) { return NewSilentError(err) } // Ctrl-C at the OPF prompt is the same gesture as declining the push diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index 28458085a9..baa58dfc14 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -32,6 +32,10 @@ import ( // prompt does, rather than an error. var ErrOPFAbortedByUser = errors.New("OPF prompt aborted by user; push cancelled") +// ErrCheckpointRefsStayQueued is PushQueuedCheckpointRefs stopping early with +// refs still queued. The flush has already printed why. +var ErrCheckpointRefsStayQueued = errors.New("checkpoint refs stay queued") + var opfPrePushProgressWriter io.Writer = os.Stderr // PrePush is called by the git pre-push hook before pushing to a remote. @@ -591,7 +595,7 @@ func PushQueuedCheckpointRefs(ctx context.Context, repo *git.Repository, remote // The flush is fail-soft about an early stop, but this caller reports // success on a nil error, so refs left behind must surface. A caller's // cancellation stays recognisable through the wrap. - err = fmt.Errorf("%d checkpoint ref(s) stay queued: %s", res.remaining, res.failureLine(nil)) + err = fmt.Errorf("%w (%d): %s", ErrCheckpointRefsStayQueued, res.remaining, res.failureLine(nil)) if ctxErr := ctx.Err(); ctxErr != nil { err = fmt.Errorf("%w: %w", err, ctxErr) } diff --git a/cmd/entire/cli/strategy/refs_push_test.go b/cmd/entire/cli/strategy/refs_push_test.go index fdeaea5719..f02e7e7aab 100644 --- a/cmd/entire/cli/strategy/refs_push_test.go +++ b/cmd/entire/cli/strategy/refs_push_test.go @@ -261,12 +261,17 @@ func TestPushQueuedCheckpointRefs_FailureLeavesRefsQueued(t *testing.T) { assert.ElementsMatch(t, refs, remaining, "failed push leaves refs queued") } -// TestPushQueuedCheckpointRefs_InterruptIsAnError: the explicit push reports -// success on a nil error, so an interrupted flush must return one that wraps -// the caller's cancellation (`doctor migrate` handles that on its own) and -// leave the refs queued. -func TestPushQueuedCheckpointRefs_InterruptIsAnError(t *testing.T) { - workDir, bareDir, refs := setupRepoWithCheckpointRefs(t) +// TestPushQueuedCheckpointRefs_EarlyStopIsAnError: the batch stops after +// consecutive failed chunks and the per-ref fallback lands every ref it +// retried, so the flush itself returns no error with refs never tried. The +// explicit push reports success on a nil error, so it must return one. +func TestPushQueuedCheckpointRefs_EarlyStopIsAnError(t *testing.T) { + shrinkRefPushChunkSize(t, 2) + workDir, bareDir, refs := setupRepoWithNCheckpointRefs(t, 8) + // Declines any push of more than one ref: every chunk fails, every per-ref + // retry lands. + hook := "#!/bin/sh\n[ \"$(wc -l)\" -gt 1 ] && exit 1\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(bareDir, "hooks", "pre-receive"), []byte(hook), 0o755)) t.Chdir(workDir) paths.ClearWorktreeRootCache() @@ -274,15 +279,13 @@ func TestPushQueuedCheckpointRefs_InterruptIsAnError(t *testing.T) { require.NoError(t, err) queue := enqueueRefs(t, repo, refs) - ctx, cancel := context.WithCancel(t.Context()) - cancel() - pushed, _, err := PushQueuedCheckpointRefs(ctx, repo, bareDir) - require.ErrorIs(t, err, context.Canceled) - assert.Equal(t, 0, pushed) + pushed, _, err := PushQueuedCheckpointRefs(t.Context(), repo, bareDir) + require.ErrorIs(t, err, ErrCheckpointRefsStayQueued) + assert.Equal(t, 4, pushed, "precondition: the batch stopped after two failed chunks and the fallback landed them") remaining, drainErr := queue.Drain() require.NoError(t, drainErr) - assert.ElementsMatch(t, refs, remaining, "unpushed refs stay queued") + assert.ElementsMatch(t, refs[4:], remaining, "untried refs stay queued") } // remoteRefFiles lists the files in the tree a ref points at on the bare remote.