feat(opf): scan in the background so git push never waits on the privacy filter

cmd/entire/cli/checkpoint/opf_span_cache.go

@@ -0,0 +1,152 @@
+package checkpoint
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/entireio/cli/cmd/entire/cli/gitdir"
+	"github.com/entireio/cli/cmd/entire/cli/jsonutil"
+	"github.com/entireio/cli/cmd/entire/cli/osroot"
+	"github.com/entireio/cli/redact"
+
+	"github.com/go-git/go-git/v6"
+)
+
+// OPFSpanCacheDirName holds OPF scan results between the background scan that
+// produces them and the rewrite that applies them. Like RedactCacheDirName it
+// sits in the git common dir, where nothing is walked into a checkpoint tree,
+// and every entry is derived data: deleting the directory only costs a rescan.
+const OPFSpanCacheDirName = "entire-opf-cache"
+
+// OPFSpanCacheMaxAge bounds how long an entry is kept after it was written.
+// A checkpoint is normally rewritten and pushed within minutes of its scan, so
+// an entry this old belongs to content that has shipped or been abandoned.
+const OPFSpanCacheMaxAge = 30 * 24 * time.Hour
+
+// opfSpanCache is the git-common-dir implementation of redact.OPFSpanCache.
+// Keys are hex SHA-256 digests produced by the redact package, so every entry
+// name is safe by construction; I/O still goes through the shared root.
+type opfSpanCache struct {
+	root *os.Root
+}
+
+// OPFSpanCacheForRepo opens the cache in repo's git common dir. It resolves the
+// directory from the repository itself, never from the process's working
+// directory, so a caller holding one repo can never read or write another's
+// scan results.
+func OPFSpanCacheForRepo(repo *git.Repository) (redact.OPFSpanCache, error) {
+	_, commonDir, err := repositoryDirs(repo)
+	if err != nil {
+		return nil, err
+	}
+	return OPFSpanCacheAt(commonDir)
+}
+
+// OPFSpanCacheAt opens the cache under gitCommonDir, creating its directory.
+func OPFSpanCacheAt(gitCommonDir string) (redact.OPFSpanCache, error) {
+	root, err := gitdir.OpenAt(gitCommonDir)
+	if err != nil {
+		return nil, fmt.Errorf("open git common dir: %w", err)
+	}
+	if err := osroot.MkdirAllNoSymlink(root, OPFSpanCacheDirName, 0o700); err != nil {
+		return nil, fmt.Errorf("create %s: %w", OPFSpanCacheDirName, err)
+	}
+	return &opfSpanCache{root: root}, nil
+}
+
+// opfSpanRecord is the stored form of one redact.Span. It is a separate,
+// tagged type so the on-disk format does not follow redact's field names.
+type opfSpanRecord struct {
+	Start int    `json:"s"`
+	End   int    `json:"e"`
+	Label string `json:"l"`
+}
+
+func opfSpanEntryName(key string) (string, error) {
+	if len(key) != 64 || strings.Trim(key, "0123456789abcdef") != "" {
+		return "", fmt.Errorf("invalid OPF span cache key %q", key)
+	}
+	return OPFSpanCacheDirName + "/" + key + ".json", nil
+}
+
+func (c *opfSpanCache) LoadOPFSpans(key string) (map[string][]redact.Span, bool) {
+	name, err := opfSpanEntryName(key)
+	if err != nil {
+		return nil, false
+	}
+	data, err := osroot.ReadFileNoFollow(c.root, name)
+	if err != nil {
+		return nil, false
+	}
+	var records map[string][]opfSpanRecord
+	if err := json.Unmarshal(data, &records); err != nil || records == nil {
+		return nil, false
+	}
+	spans := make(map[string][]redact.Span, len(records))
+	for leaf, recs := range records {
+		out := make([]redact.Span, 0, len(recs))
+		for _, r := range recs {
+			out = append(out, redact.Span{Start: r.Start, End: r.End, Label: r.Label})
+		}
+		spans[leaf] = out
+	}
+	return spans, true
+}
+
+func (c *opfSpanCache) StoreOPFSpans(key string, spans map[string][]redact.Span) error {
+	name, err := opfSpanEntryName(key)
+	if err != nil {
+		return err
+	}
+	records := make(map[string][]opfSpanRecord, len(spans))
+	for leaf, ss := range spans {
+		recs := make([]opfSpanRecord, 0, len(ss))
+		for _, sp := range ss {
+			recs = append(recs, opfSpanRecord{Start: sp.Start, End: sp.End, Label: sp.Label})
+		}
+		records[leaf] = recs
+	}
+	data, err := json.Marshal(records)
+	if err != nil {
+		return fmt.Errorf("encode OPF span cache entry: %w", err)
+	}
+	if err := jsonutil.WriteFileAtomicIn(c.root, name, data, 0o600); err != nil {
+		return fmt.Errorf("write OPF span cache entry: %w", err)
+	}
+	return nil
+}
+
+// PruneOPFSpanCache removes entries written before now-maxAge and returns how
+// many it removed. Best-effort: entries it cannot inspect are left in place.
+func PruneOPFSpanCache(gitCommonDir string, now time.Time, maxAge time.Duration) (int, error) {
+	root, err := gitdir.OpenAt(gitCommonDir)
+	if err != nil {
+		return 0, fmt.Errorf("open git common dir: %w", err)
+	}
+	entries, err := osroot.ReadDirNoSymlinks(root, OPFSpanCacheDirName)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return 0, nil
+		}
+		return 0, fmt.Errorf("list %s: %w", OPFSpanCacheDirName, err)
+	}
+	removed := 0
+	for _, e := range entries {
+		if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".json") {
+			continue
+		}
+		info, infoErr := e.Info()
+		if infoErr != nil || now.Sub(info.ModTime()) < maxAge {
+			continue
+		}
+		if osroot.RemoveNoSymlinks(root, OPFSpanCacheDirName+"/"+e.Name()) == nil {
+			removed++
+		}
+	}
+	return removed, nil
+}

cmd/entire/cli/checkpoint/opf_span_cache_test.go

@@ -0,0 +1,104 @@
+package checkpoint
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/entireio/cli/cmd/entire/cli/gitrepo"
+	"github.com/entireio/cli/cmd/entire/cli/testutil"
+	"github.com/entireio/cli/redact"
+)
+
+var testOPFKey = strings.Repeat("ab", 32)
+
+func TestOPFSpanCache_RoundTrip(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	cache, err := OPFSpanCacheAt(dir)
+	require.NoError(t, err)
+
+	_, ok := cache.LoadOPFSpans(testOPFKey)
+	assert.False(t, ok, "a missing entry must read as not cached")
+
+	want := map[string][]redact.Span{
+		strings.Repeat("cd", 32): {{Start: 0, End: 5, Label: "private_person"}},
+		strings.Repeat("ef", 32): {},
+	}
+	require.NoError(t, cache.StoreOPFSpans(testOPFKey, want))
+	got, ok := cache.LoadOPFSpans(testOPFKey)
+	require.True(t, ok)
+	assert.Equal(t, want[strings.Repeat("cd", 32)], got[strings.Repeat("cd", 32)])
+	_, present := got[strings.Repeat("ef", 32)]
+	assert.True(t, present, "a leaf with no spans must still be recorded as scanned")
+
+	info, err := os.Stat(filepath.Join(dir, OPFSpanCacheDirName, testOPFKey+".json"))
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o600), info.Mode().Perm())
+}
+
+func TestOPFSpanCache_RejectsKeysThatAreNotDigests(t *testing.T) {
+	t.Parallel()
+	cache, err := OPFSpanCacheAt(t.TempDir())
+	require.NoError(t, err)
+	for _, key := range []string{"", "../escape", strings.Repeat("A", 64), strings.Repeat("a", 63)} {
+		require.Error(t, cache.StoreOPFSpans(key, nil), "key %q", key)
+		_, ok := cache.LoadOPFSpans(key)
+		assert.False(t, ok, "key %q", key)
+	}
+}
+
+func TestOPFSpanCache_CorruptEntryReadsAsMissing(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	cache, err := OPFSpanCacheAt(dir)
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(filepath.Join(dir, OPFSpanCacheDirName, testOPFKey+".json"), []byte("{not json"), 0o600))
+	_, ok := cache.LoadOPFSpans(testOPFKey)
+	assert.False(t, ok)
+}
+
+func TestPruneOPFSpanCache_RemovesOnlyOldEntries(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	cache, err := OPFSpanCacheAt(dir)
+	require.NoError(t, err)
+	oldKey, newKey := strings.Repeat("0a", 32), strings.Repeat("0b", 32)
+	require.NoError(t, cache.StoreOPFSpans(oldKey, nil))
+	require.NoError(t, cache.StoreOPFSpans(newKey, nil))
+	now := time.Now()
+	oldPath := filepath.Join(dir, OPFSpanCacheDirName, oldKey+".json")
+	require.NoError(t, os.Chtimes(oldPath, now.Add(-2*OPFSpanCacheMaxAge), now.Add(-2*OPFSpanCacheMaxAge)))
+
+	removed, err := PruneOPFSpanCache(dir, now, OPFSpanCacheMaxAge)
+	require.NoError(t, err)
+	assert.Equal(t, 1, removed)
+	_, ok := cache.LoadOPFSpans(oldKey)
+	assert.False(t, ok, "the stale entry must be gone")
+	_, ok = cache.LoadOPFSpans(newKey)
+	assert.True(t, ok, "a recent entry must survive")
+}
+
+// The cache must live in the repository it was opened for, whatever the
+// process's working directory is. Resolving it from the working directory once
+// wrote test results into the developer's own checkout.
+func TestOPFSpanCacheForRepo_UsesTheRepositoryNotTheWorkingDirectory(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+	testutil.InitRepo(t, dir)
+	repo, err := gitrepo.OpenPath(dir)
+	require.NoError(t, err)
+	t.Cleanup(func() { repo.Close() })
+
+	cache, err := OPFSpanCacheForRepo(repo)
+	require.NoError(t, err)
+	require.NoError(t, cache.StoreOPFSpans(testOPFKey, nil))
+
+	_, err = os.Stat(filepath.Join(dir, ".git", OPFSpanCacheDirName, testOPFKey+".json"))
+	require.NoError(t, err, "the entry must be written under the repository's own git dir")
+}

cmd/entire/cli/execx/spawn_detached.go

@@ -2,7 +2,6 @@ package execx
 
 import (
 	"context"
-	"io"
 	"os"
 	"os/exec"
 	"testing"
@@ -13,7 +12,7 @@ import (
 // CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS on Windows, via detachFromTTY).
 // The child runs in dir (os.TempDir() when empty, so the child never holds the
 // parent's working directory), inherits the parent's environment, and has its
-// stdout/stderr discarded. Best-effort: every error is swallowed — callers
+// stdout/stderr sent to the null device. Best-effort: every error is swallowed — callers
 // treat the spawn as advisory background work.
 //
 // In-process `go test` runs are a no-op: the current executable is the test
@@ -28,6 +27,25 @@ func SpawnDetached(dir string, args ...string) {
 		return
 	}
 
+	cmd := detachedCommand(executable, dir, args...)
+	if err := cmd.Start(); err != nil {
+		return
+	}
+	// Release the process so it can run independently of the parent.
+	//nolint:errcheck // best effort — the child continues regardless
+	_ = cmd.Process.Release()
+}
+
+// detachedCommand builds the child SpawnDetached starts. Separate from the
+// spawn so tests can check how the child is wired without forking it.
+//
+// Stdout and stderr stay nil, which os/exec opens as the null device. A
+// non-*os.File writer such as io.Discard would instead hand the child a pipe
+// drained by a goroutine in this process; once this process exits, the child's
+// next write to that pipe raises SIGPIPE and kills it. A child that writes
+// anything to stderr, such as a warning from the `__opf_scan` worker, would die
+// on its first line.
+func detachedCommand(executable, dir string, args ...string) *exec.Cmd {
 	// context.Background(): the child must outlive the parent, so it is never
 	// tied to a cancellable context.
 	cmd := exec.CommandContext(context.Background(), executable, args...)
@@ -37,13 +55,5 @@ func SpawnDetached(dir string, args ...string) {
 		cmd.Dir = os.TempDir()
 	}
 	cmd.Env = os.Environ()
-	cmd.Stdout = io.Discard
-	cmd.Stderr = io.Discard
-
-	if err := cmd.Start(); err != nil {
-		return
-	}
-	// Release the process so it can run independently of the parent.
-	//nolint:errcheck // best effort — the child continues regardless
-	_ = cmd.Process.Release()
+	return cmd
 }

cmd/entire/cli/execx/spawn_detached_test.go

@@ -0,0 +1,25 @@
+package execx
+
+import (
+	"testing"
+)
+
+// A detached child must not write into a pipe this process drains: after the
+// parent exits, the child's next stdout/stderr write raises SIGPIPE and kills
+// it. Nil streams are opened as the null device instead.
+func TestDetachedCommand_StdioIsNullDevice(t *testing.T) {
+	t.Parallel()
+	cmd := detachedCommand("/bin/entire", "", "__opf_scan")
+	if cmd.Stdout != nil {
+		t.Errorf("Stdout = %T; want nil so the child writes to the null device, not a pipe", cmd.Stdout)
+	}
+	if cmd.Stderr != nil {
+		t.Errorf("Stderr = %T; want nil so the child writes to the null device, not a pipe", cmd.Stderr)
+	}
+	if cmd.Stdin != nil {
+		t.Errorf("Stdin = %T; want nil", cmd.Stdin)
+	}
+	if cmd.Dir == "" {
+		t.Error("Dir is empty; want os.TempDir() fallback")
+	}
+}

cmd/entire/cli/hooks_git_cmd.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"os"
 	"time"
 
 	"github.com/entireio/cli/cmd/entire/cli/agent/external"
@@ -228,7 +229,7 @@ func newHooksGitPostRewriteCmd() *cobra.Command {
 }
 
 func newHooksGitPrePushCmd() *cobra.Command {
-	return &cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "pre-push <remote>",
 		Short: "Handle pre-push git hook",
 		Args:  cobra.ExactArgs(1),
@@ -250,7 +251,16 @@ func newHooksGitPrePushCmd() *cobra.Command {
 			defer g.span.End()
 			g.logInvoked(slog.String("remote", remote))
 
-			hookErr := g.strategy.PrePushFromGitHook(g.ctx, remote)
+			ctx := g.ctx
+			// Only scripts that replay stdin to whatever runs after them set
+			// this (see strategy.PrePushStdinRefsEnv).
+			if os.Getenv(strategy.PrePushStdinRefsEnv) == "1" {
+				// The ref list git passes on stdin is what lets OPF refuse a
+				// push that sends unverified checkpoint content itself.
+				ctx = strategy.WithPrePushRefsFrom(ctx, cmd.InOrStdin())
+			}
+
+			hookErr := g.strategy.PrePushFromGitHook(ctx, remote)
 			g.logCompleted(hookErr)
 
 			// Propagate the error so the hook script exits non-zero and
@@ -270,4 +280,5 @@ func newHooksGitPrePushCmd() *cobra.Command {
 			return fmt.Errorf("pre-push: %w", hookErr)
 		},
 	}
+	return cmd
 }