From a55ec27273473ff3f91a2ce2a8873b0d67fb244b Mon Sep 17 00:00:00 2001 From: rwrife Date: Sat, 11 Jul 2026 20:46:17 +0000 Subject: [PATCH] Add sp dedup: content-hash duplicate detection (#28) --- README.md | 39 +++++- internal/cli/dedup.go | 114 +++++++++++++++++ internal/cli/dedup_test.go | 132 ++++++++++++++++++++ internal/cli/root.go | 1 + internal/render/dedup.go | 213 ++++++++++++++++++++++++++++++++ internal/render/dedup_test.go | 159 ++++++++++++++++++++++++ internal/store/dedup.go | 191 +++++++++++++++++++++++++++++ internal/store/dedup_test.go | 224 ++++++++++++++++++++++++++++++++++ 8 files changed, 1072 insertions(+), 1 deletion(-) create mode 100644 internal/cli/dedup.go create mode 100644 internal/cli/dedup_test.go create mode 100644 internal/render/dedup.go create mode 100644 internal/render/dedup_test.go create mode 100644 internal/store/dedup.go create mode 100644 internal/store/dedup_test.go diff --git a/README.md b/README.md index 419f530..bbc5727 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ sp doctor --json | jq -e '.healthy' # gate a script on store health sp ls --json | jq '.[].id' # machine-readable output for scripting sp completion zsh > "${fpath[1]}/_sp" # tab-completion for your shell sp scan # tripwire: does this scratch hold a secret? +sp dedup # report byte-identical scratches (read-only) +sp dedup --collapse # send redundant copies to the morgue (morgue-first) sp resurrect # changed your mind? pull it back sp promote # the good ones: graduate a scratch into your repo ``` @@ -40,7 +42,9 @@ scripting polish — **`sp ls --json`** / **`sp doctor --json`** and **`sp completion`** for bash/zsh/fish; **`sp promote`** to graduate a scratch into your repo; and a **secret tripwire** — **`sp scan`** flags scratches that look like they hold credentials, `sp ls` marks them with a 🔑, and `sp promote` -refuses them unless you pass `--allow-secrets` (M6, in progress). +refuses them unless you pass `--allow-secrets`; and **content dedup** — +**`sp dedup`** reports byte-identical scratches and `--collapse` sweeps the +redundant copies to the morgue (M6, in progress). ### `sp new [name]` @@ -319,6 +323,39 @@ The tripwire also shows up where it matters most: you pass `--allow-secrets` — the last line of defense before a leaked key lands somewhere it might get committed. +### `sp dedup` — find (and collapse) byte-identical scratches + +Throwaway files breed duplicates: the same log pasted three times, an agent +re-capturing identical output on every run, the same snippet `sp new`'d across +two projects. Each copy ages out separately and quietly wastes footprint. +`sp dedup` hashes every **live** scratch's content (SHA-256), groups the +byte-identical copies into clusters, and reports how much they cost — naming the +oldest copy in each cluster as the **canonical** one to keep. + +This is a content-equality primitive, distinct from `sp doctor` (which +reconciles index-vs-disk drift, not equality) and `sp reap` (TTL-based). Two +scratches with different names, tags, or extensions but identical bytes are +still duplicates. + +```bash +sp dedup # read-only report of duplicate clusters (canonical + wasted bytes) +sp dedup --no-color # plain, script-friendly +sp dedup --json # stable JSON object (no color, no flavor) +sp dedup --collapse # move the redundant copies to the morgue, keep the canonical +``` + +By default `sp dedup` is **strictly read-only** — it moves nothing. Pass +`--collapse` to send the redundant copies to the morgue, keeping each cluster's +canonical (oldest) member live. Like `sp rm`, collapse is **morgue-first and +never hard-deletes**: collapsed copies are recoverable with `sp resurrect` until +the reaper purges them past the grace window. + +A unique store prints a clean, one-line bill of health. The `--json` form +carries a top-level `clean` flag plus `clusters` (always an array, never null; +each cluster has its full `digest`, `wastedBytes`, and `members` with a +`canonical` flag) and a `collapsed` object that is `null` until a `--collapse` +run moves something: `sp dedup --json | jq -e '.clean'`. + ### `sp completion ` Prints a shell completion script to stdout so `sp`'s commands and flags diff --git a/internal/cli/dedup.go b/internal/cli/dedup.go new file mode 100644 index 0000000..5f6ee75 --- /dev/null +++ b/internal/cli/dedup.go @@ -0,0 +1,114 @@ +package cli + +import ( + "github.com/spf13/cobra" + + "github.com/rwrife/scratchpatch/internal/render" + "github.com/rwrife/scratchpatch/internal/store" +) + +func newDedupCommand() *cobra.Command { + var noColor bool + var asJSON bool + var collapse bool + + cmd := &cobra.Command{ + Use: "dedup", + Short: "Find (and optionally collapse) byte-identical scratches", + Long: "Throwaway files breed duplicates: the same log pasted three times, an\n" + + "agent re-capturing identical output on every run, the same snippet\n" + + "`sp new`'d across two projects. dedup hashes every live scratch's content\n" + + "and groups the byte-identical copies into clusters.\n\n" + + "By default dedup is strictly read-only: it reports the clusters, names the\n" + + "oldest copy as canonical, and shows how many bytes the redundant copies\n" + + "waste — it moves nothing.\n\n" + + "Pass --collapse to send the redundant copies to the morgue, keeping each\n" + + "cluster's canonical (oldest) member live. Like `sp rm`, this never\n" + + "hard-deletes: collapsed copies are recoverable with `sp resurrect` until\n" + + "the reaper purges them past the grace window.\n\n" + + "dedup is content-equality only — distinct from `sp doctor` (index-vs-disk\n" + + "drift) and `sp reap` (TTL-based). Pass --json for a stable, machine-\n" + + "readable object (no color, no flavor): `sp dedup --json | jq '.clean'`.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runDedup(cmd, noColor, asJSON, collapse) + }, + } + + cmd.Flags().BoolVar(&noColor, "no-color", false, "force plain output even on a TTY") + cmd.Flags().BoolVar(&asJSON, "json", false, "emit a JSON object instead of a report (for scripting)") + cmd.Flags().BoolVar(&collapse, "collapse", false, "move redundant copies to the morgue, keeping the oldest as canonical") + + return cmd +} + +func runDedup(cmd *cobra.Command, noColor, asJSON, collapse bool) error { + st, err := store.Open() + if err != nil { + return err + } + + report, err := st.Dedup() + if err != nil { + return err + } + + data := toDedupData(report) + + // --collapse moves the redundant copies to the morgue, then records the + // outcome on the report so both the human and JSON views can confirm it. + if collapse && !report.Clean() { + res, err := st.Collapse(report) + if err != nil { + return err + } + data.Collapsed = &render.DedupCollapsedData{ + MovedIDs: res.MovedIDs, + ReclaimedBytes: res.ReclaimedBytes, + } + } else if collapse { + // Nothing to collapse, but the user asked — report an empty outcome so + // the message is "collapsed 0" rather than the read-only footer. + data.Collapsed = &render.DedupCollapsedData{} + } + + out := cmd.OutOrStdout() + + // --json is intentionally color-free and personality-free; never tint it, + // regardless of TTY or --no-color, matching `sp ls --json`/`sp doctor --json`. + if asJSON { + return render.DedupReportJSON(out, data) + } + + color := !noColor && isTerminal(out) + return render.DedupReport(out, data, color) +} + +// toDedupData flattens the store's DedupReport into the render layer's plain +// view, so render never has to import the store package — the same adapter +// pattern doctor.go/reap.go use. +func toDedupData(r store.DedupReport) render.DedupData { + clusters := make([]render.DedupClusterData, 0, len(r.Clusters)) + for _, c := range r.Clusters { + members := make([]render.DedupMemberData, 0, len(c.Members)) + for _, m := range c.Members { + members = append(members, render.DedupMemberData{ + ID: m.Scratch.ID, + Name: m.Scratch.Name, + Size: m.Scratch.Size, + CreatedAt: m.Scratch.CreatedAt, + Canonical: m.Canonical, + }) + } + clusters = append(clusters, render.DedupClusterData{ + Digest: c.Digest, + Members: members, + WastedBytes: c.WastedBytes, + }) + } + return render.DedupData{ + Clusters: clusters, + TotalWasted: r.TotalWasted, + ScannedCount: r.ScannedCount, + } +} diff --git a/internal/cli/dedup_test.go b/internal/cli/dedup_test.go new file mode 100644 index 0000000..442c694 --- /dev/null +++ b/internal/cli/dedup_test.go @@ -0,0 +1,132 @@ +package cli + +import ( + "encoding/json" + "path/filepath" + "strings" + "testing" +) + +// TestDedupCleanStoreCLI: a store of unique scratches reports no duplicates. +func TestDedupCleanStoreCLI(t *testing.T) { + s := newSession(t) + if out, err := s.run("new", "a", "--content", "alpha\n", "--ext", "txt"); err != nil { + t.Fatalf("new a: %v (%s)", err, out) + } + if out, err := s.run("new", "b", "--content", "beta\n", "--ext", "txt"); err != nil { + t.Fatalf("new b: %v (%s)", err, out) + } + + out, err := s.run("dedup", "--no-color") + if err != nil { + t.Fatalf("dedup: %v (%s)", err, out) + } + if !strings.Contains(out, "no duplicates") { + t.Errorf("clean store should report no duplicates; got %q", out) + } +} + +// TestDedupReportsAndCollapsesCLI: identical content is reported as a cluster; +// default is read-only; --collapse morgues the redundant copy (never hard- +// deletes), keeping the canonical live. +func TestDedupReportsAndCollapsesCLI(t *testing.T) { + s := newSession(t) + body := "duplicate body\n" + for _, name := range []string{"first", "second", "third"} { + if out, err := s.run("new", name, "--content", body, "--ext", "txt"); err != nil { + t.Fatalf("new %s: %v (%s)", name, err, out) + } + } + + // Read-only report first. + out, err := s.run("dedup", "--no-color") + if err != nil { + t.Fatalf("dedup: %v (%s)", err, out) + } + if !strings.Contains(out, "cluster") || !strings.Contains(out, "canonical") { + t.Errorf("dedup should report a cluster with a canonical; got %q", out) + } + if !strings.Contains(out, "--collapse") { + t.Errorf("read-only dedup should point at --collapse; got %q", out) + } + // Read-only: still 3 live scratch files on disk. + if got := len(globTxt(t, s.home)); got != 3 { + t.Fatalf("read-only dedup changed the store: %d live files, want 3", got) + } + + // Now collapse. + out, err = s.run("dedup", "--collapse", "--no-color") + if err != nil { + t.Fatalf("dedup --collapse: %v (%s)", err, out) + } + if !strings.Contains(out, "collapsed") || !strings.Contains(out, "morgue") { + t.Errorf("collapse should confirm the move; got %q", out) + } + + // Canonical stays live; the two redundant copies are morgued (not gone). + live := globTxt(t, filepath.Join(s.home, "scratches")) + if len(live) != 1 { + t.Errorf("after collapse: %d live files, want 1", len(live)) + } + morgue := globTxt(t, filepath.Join(s.home, "morgue")) + if len(morgue) != 2 { + t.Errorf("after collapse: %d morgue files, want 2 (nothing hard-deleted)", len(morgue)) + } +} + +// TestDedupJSONCLI: --json is a stable, colorless object with the clean flag +// and cluster data. +func TestDedupJSONCLI(t *testing.T) { + s := newSession(t) + body := "same\n" + for _, name := range []string{"x", "y"} { + if out, err := s.run("new", name, "--content", body, "--ext", "txt"); err != nil { + t.Fatalf("new %s: %v (%s)", name, err, out) + } + } + + out, err := s.run("dedup", "--json") + if err != nil { + t.Fatalf("dedup --json: %v (%s)", err, out) + } + if strings.Contains(out, "\x1b[") { + t.Errorf("dedup --json must be colorless; got %q", out) + } + + var rec struct { + Clean bool `json:"clean"` + ClusterCount int `json:"clusterCount"` + Clusters []struct { + Count int `json:"count"` + Members []struct { + Canonical bool `json:"canonical"` + } `json:"members"` + } `json:"clusters"` + Collapsed any `json:"collapsed"` + } + if err := json.Unmarshal([]byte(out), &rec); err != nil { + t.Fatalf("invalid JSON: %v (%q)", err, out) + } + if rec.Clean { + t.Errorf("two identical scratches should report clean=false") + } + if rec.ClusterCount != 1 || len(rec.Clusters) != 1 || rec.Clusters[0].Count != 2 { + t.Fatalf("unexpected cluster shape: %+v", rec) + } + if rec.Collapsed != nil { + t.Errorf("collapsed should be null without --collapse; got %v", rec.Collapsed) + } +} + +// globTxt returns the *.txt files under dir (or under dir/scratches when dir is +// a store home passed bare). +func globTxt(t *testing.T, dir string) []string { + t.Helper() + // If dir is a store home (contains scratches/), glob live scratches. + pattern := filepath.Join(dir, "*.txt") + matches, _ := filepath.Glob(pattern) + if len(matches) == 0 { + matches, _ = filepath.Glob(filepath.Join(dir, "scratches", "*.txt")) + } + return matches +} diff --git a/internal/cli/root.go b/internal/cli/root.go index ed6ef8c..a655788 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -54,6 +54,7 @@ func NewRootCommand() *cobra.Command { newPromoteCommand(), newReapCommand(), newDoctorCommand(), + newDedupCommand(), newScanCommand(), newCompletionCommand(), ) diff --git a/internal/render/dedup.go b/internal/render/dedup.go new file mode 100644 index 0000000..417cc93 --- /dev/null +++ b/internal/render/dedup.go @@ -0,0 +1,213 @@ +// dedup.go renders `sp dedup`: the human, tombstone-flavored duplicate report +// and its color/personality-free `--json` counterpart. +// +// Like the doctor views, render never imports the store package. The cli layer +// flattens store.DedupReport into the plain DedupData below, so this file works +// purely from render-owned types. The JSON path carries no color and no +// flavor — same stable-contract rule as `sp ls --json` / `sp doctor --json`. +package render + +import ( + "fmt" + "io" + "strings" + "time" +) + +// DedupMemberData is one scratch in a duplicate cluster, flattened for render. +type DedupMemberData struct { + ID string + Name string + Size int64 + CreatedAt time.Time + Canonical bool +} + +// DedupClusterData is a group of byte-identical scratches. Members are ordered +// canonical-first. WastedBytes is what the redundant copies cost. +type DedupClusterData struct { + Digest string + Members []DedupMemberData + WastedBytes int64 +} + +// DedupData is the plain summary `sp dedup` renders. +type DedupData struct { + Clusters []DedupClusterData + TotalWasted int64 + ScannedCount int + // Collapsed, when non-nil, means a --collapse run happened: it names what + // moved to the morgue so the report can confirm the action. + Collapsed *DedupCollapsedData +} + +// DedupCollapsedData records the outcome of a --collapse run for the report. +type DedupCollapsedData struct { + MovedIDs []string + ReclaimedBytes int64 +} + +func (d DedupData) clean() bool { return len(d.Clusters) == 0 } + +// countClusters pluralizes a cluster count for the human report. +func countClusters(n int) string { + if n == 1 { + return "1 cluster" + } + return fmt.Sprintf("%d clusters", n) +} + +// shortDigest trims a full sha256 to a glanceable prefix for the human report; +// the JSON path keeps the full digest. +func shortDigest(digest string) string { + if len(digest) > 12 { + return digest[:12] + } + return digest +} + +// DedupReport writes the human, tombstone-flavored duplicate report to w. +// A unique store gets a clean, reassuring headline; a store with duplicates +// gets the clusters, each showing the canonical keep-copy and the redundant +// extras with the bytes they waste. When d.Collapsed is set, it also confirms +// what was moved to the morgue. +func DedupReport(w io.Writer, d DedupData, color bool) error { + pal := defaultPalette() + var b strings.Builder + + if d.clean() { + headline := fmt.Sprintf("no duplicates — every one of the %s living scratches is one of a kind", + countScratches(d.ScannedCount)) + writeLine(&b, headline, color, pal.fresh) + _, err := io.WriteString(w, b.String()) + return err + } + + headline := fmt.Sprintf("found %s of identical scratches — %s haunting the store in duplicate", + countClusters(len(d.Clusters)), humanSize(d.TotalWasted)) + writeLine(&b, headline, color, pal.header) + writeLine(&b, fmt.Sprintf("scanned %s live", countScratches(d.ScannedCount)), color, pal.header) + + for _, c := range d.Clusters { + fmt.Fprintf(&b, "\ncluster %s — %d copies, %s reclaimable:\n", + shortDigest(c.Digest), len(c.Members), humanSize(c.WastedBytes)) + for _, m := range c.Members { + if m.Canonical { + line := fmt.Sprintf(" %s %s %s (canonical — the original, kept)", + m.ID, nameOrDash(m.Name), humanSize(m.Size)) + writeLine(&b, line, color, pal.fresh) + continue + } + line := fmt.Sprintf(" %s %s %s (redundant)", + m.ID, nameOrDash(m.Name), humanSize(m.Size)) + writeLine(&b, line, color, pal.soon) + } + } + + if d.Collapsed != nil { + fmt.Fprintf(&b, "\ncollapsed %s to the morgue, %s reclaimed. "+ + "Nothing was hard-deleted — `sp resurrect` brings any copy back.\n", + countScratches(len(d.Collapsed.MovedIDs)), humanSize(d.Collapsed.ReclaimedBytes)) + } else { + fmt.Fprintf(&b, "\ndedup only reports; nothing was moved. "+ + "Re-run with --collapse to send the redundant copies to the morgue (never hard-deleted).\n") + } + + _, err := io.WriteString(w, b.String()) + return err +} + +// --- JSON views ------------------------------------------------------------ + +// DedupJSON is the scriptable record for `sp dedup --json`. Slices are always +// non-nil so the shape never flips to null on a clean store, mirroring the ls / +// doctor JSON contracts. No color, no flavor — pure data. +type DedupJSON struct { + // Clean is the one-field summary so a script can gate on `.clean`. + Clean bool `json:"clean"` + ScannedCount int `json:"scannedCount"` + ClusterCount int `json:"clusterCount"` + TotalWasted int64 `json:"totalWasted"` + TotalWastedHuman string `json:"totalWastedHuman"` + Clusters []DedupClusterJSON `json:"clusters"` + // Collapsed is null unless a --collapse run happened, in which case it + // reports what moved to the morgue. + Collapsed *DedupCollapsedJSON `json:"collapsed"` +} + +// DedupClusterJSON is the scriptable view of one duplicate cluster. +type DedupClusterJSON struct { + Digest string `json:"digest"` + Count int `json:"count"` + WastedBytes int64 `json:"wastedBytes"` + WastedHuman string `json:"wastedHuman"` + Members []DedupMemberJSON `json:"members"` +} + +// DedupMemberJSON is the scriptable view of one cluster member. +type DedupMemberJSON struct { + ID string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + SizeHuman string `json:"sizeHuman"` + CreatedAt time.Time `json:"createdAt"` + Canonical bool `json:"canonical"` +} + +// DedupCollapsedJSON is the scriptable view of a --collapse outcome. +type DedupCollapsedJSON struct { + MovedIDs []string `json:"movedIds"` + ReclaimedBytes int64 `json:"reclaimedBytes"` + ReclaimedHuman string `json:"reclaimedHuman"` +} + +// DedupReportJSON writes a DedupData to w as a single DedupJSON object. +// Like the other --json paths it is intentionally color- and personality-free, +// and every slice is emitted as an array (never null). +func DedupReportJSON(w io.Writer, d DedupData) error { + clusters := make([]DedupClusterJSON, 0, len(d.Clusters)) + for _, c := range d.Clusters { + members := make([]DedupMemberJSON, 0, len(c.Members)) + for _, m := range c.Members { + members = append(members, DedupMemberJSON{ + ID: m.ID, + Name: m.Name, + Size: m.Size, + SizeHuman: humanSize(m.Size), + CreatedAt: m.CreatedAt, + Canonical: m.Canonical, + }) + } + clusters = append(clusters, DedupClusterJSON{ + Digest: c.Digest, + Count: len(c.Members), + WastedBytes: c.WastedBytes, + WastedHuman: humanSize(c.WastedBytes), + Members: members, + }) + } + + var collapsed *DedupCollapsedJSON + if d.Collapsed != nil { + moved := d.Collapsed.MovedIDs + if moved == nil { + moved = []string{} + } + collapsed = &DedupCollapsedJSON{ + MovedIDs: moved, + ReclaimedBytes: d.Collapsed.ReclaimedBytes, + ReclaimedHuman: humanSize(d.Collapsed.ReclaimedBytes), + } + } + + rec := DedupJSON{ + Clean: d.clean(), + ScannedCount: d.ScannedCount, + ClusterCount: len(d.Clusters), + TotalWasted: d.TotalWasted, + TotalWastedHuman: humanSize(d.TotalWasted), + Clusters: clusters, + Collapsed: collapsed, + } + return writeJSON(w, rec) +} diff --git a/internal/render/dedup_test.go b/internal/render/dedup_test.go new file mode 100644 index 0000000..9a260de --- /dev/null +++ b/internal/render/dedup_test.go @@ -0,0 +1,159 @@ +package render + +import ( + "bytes" + "encoding/json" + "testing" + "time" +) + +func sampleDedupData() DedupData { + created := time.Now().Add(-time.Hour) + return DedupData{ + ScannedCount: 4, + TotalWasted: 20, + Clusters: []DedupClusterData{ + { + Digest: "abc123def456abc123def456", + WastedBytes: 20, + Members: []DedupMemberData{ + {ID: "old00001", Name: "original", Size: 20, CreatedAt: created, Canonical: true}, + {ID: "new00002", Name: "copy", Size: 20, CreatedAt: created.Add(time.Minute)}, + }, + }, + }, + } +} + +// TestDedupReportCleanStore: a unique store gets a reassuring, cluster-free +// headline and the read-only footer. +func TestDedupReportCleanStore(t *testing.T) { + var buf bytes.Buffer + if err := DedupReport(&buf, DedupData{ScannedCount: 5}, false); err != nil { + t.Fatalf("DedupReport: %v", err) + } + out := buf.String() + if !bytes.Contains(buf.Bytes(), []byte("no duplicates")) { + t.Errorf("clean report should say no duplicates; got %q", out) + } +} + +// TestDedupReportListsClusters: a store with duplicates names the canonical +// keep-copy and the redundant extras, and (read-only) points at --collapse. +func TestDedupReportListsClusters(t *testing.T) { + var buf bytes.Buffer + if err := DedupReport(&buf, sampleDedupData(), false); err != nil { + t.Fatalf("DedupReport: %v", err) + } + out := buf.String() + for _, want := range []string{"old00001", "new00002", "canonical", "redundant", "--collapse"} { + if !bytes.Contains(buf.Bytes(), []byte(want)) { + t.Errorf("report missing %q; got:\n%s", want, out) + } + } +} + +// TestDedupReportCollapseConfirmation: after a collapse the report confirms +// what moved to the morgue instead of the read-only footer. +func TestDedupReportCollapseConfirmation(t *testing.T) { + data := sampleDedupData() + data.Collapsed = &DedupCollapsedData{MovedIDs: []string{"new00002"}, ReclaimedBytes: 20} + + var buf bytes.Buffer + if err := DedupReport(&buf, data, false); err != nil { + t.Fatalf("DedupReport: %v", err) + } + if !bytes.Contains(buf.Bytes(), []byte("collapsed")) || !bytes.Contains(buf.Bytes(), []byte("morgue")) { + t.Errorf("collapse report should confirm the move; got %q", buf.String()) + } + if bytes.Contains(buf.Bytes(), []byte("--collapse to send")) { + t.Errorf("collapse report should not still suggest --collapse; got %q", buf.String()) + } +} + +// TestDedupReportJSONShape: --json is colorless, personality-free, and carries +// the stable contract (clean flag, non-nil arrays, full digest, collapsed null). +func TestDedupReportJSONShape(t *testing.T) { + var buf bytes.Buffer + if err := DedupReportJSON(&buf, sampleDedupData()); err != nil { + t.Fatalf("DedupReportJSON: %v", err) + } + if bytes.Contains(buf.Bytes(), []byte("\x1b[")) { + t.Errorf("dedup --json must be colorless; got %q", buf.String()) + } + for _, flavor := range []string{"canonical — the original", "haunting", "reclaimable"} { + if bytes.Contains(buf.Bytes(), []byte(flavor)) { + t.Errorf("dedup --json must be personality-free; found %q", flavor) + } + } + + var rec DedupJSON + if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { + t.Fatalf("invalid JSON: %v (%q)", err, buf.String()) + } + if rec.Clean { + t.Errorf("store with a cluster must report clean=false") + } + if rec.ClusterCount != 1 || rec.ScannedCount != 4 { + t.Errorf("clusterCount=%d scannedCount=%d, want 1/4", rec.ClusterCount, rec.ScannedCount) + } + if rec.TotalWasted != 20 { + t.Errorf("totalWasted=%d, want 20", rec.TotalWasted) + } + if len(rec.Clusters) != 1 || rec.Clusters[0].Count != 2 { + t.Fatalf("clusters shape wrong: %+v", rec.Clusters) + } + if rec.Clusters[0].Digest != "abc123def456abc123def456" { + t.Errorf("json should carry the full digest, got %q", rec.Clusters[0].Digest) + } + if !rec.Clusters[0].Members[0].Canonical || rec.Clusters[0].Members[1].Canonical { + t.Errorf("member canonical flags wrong: %+v", rec.Clusters[0].Members) + } + if rec.Collapsed != nil { + t.Errorf("collapsed should be null when no --collapse ran; got %+v", rec.Collapsed) + } +} + +// TestDedupReportJSONCleanArrays: a clean store still emits a non-null clusters +// array so scripts can iterate unconditionally. +func TestDedupReportJSONCleanArrays(t *testing.T) { + var buf bytes.Buffer + if err := DedupReportJSON(&buf, DedupData{ScannedCount: 2}); err != nil { + t.Fatalf("DedupReportJSON: %v", err) + } + var rec DedupJSON + if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if !rec.Clean { + t.Errorf("empty-cluster store should report clean=true") + } + if rec.Clusters == nil || len(rec.Clusters) != 0 { + t.Errorf("clusters should be empty non-nil array; got %#v", rec.Clusters) + } +} + +// TestDedupReportJSONCollapsed: a collapse run surfaces the collapsed object +// with the moved ids and reclaimed bytes. +func TestDedupReportJSONCollapsed(t *testing.T) { + data := sampleDedupData() + data.Collapsed = &DedupCollapsedData{MovedIDs: []string{"new00002"}, ReclaimedBytes: 20} + + var buf bytes.Buffer + if err := DedupReportJSON(&buf, data); err != nil { + t.Fatalf("DedupReportJSON: %v", err) + } + var rec DedupJSON + if err := json.Unmarshal(buf.Bytes(), &rec); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if rec.Collapsed == nil { + t.Fatalf("collapsed should be populated after a collapse run") + } + if len(rec.Collapsed.MovedIDs) != 1 || rec.Collapsed.MovedIDs[0] != "new00002" { + t.Errorf("movedIds = %v, want [new00002]", rec.Collapsed.MovedIDs) + } + if rec.Collapsed.ReclaimedBytes != 20 { + t.Errorf("reclaimedBytes = %d, want 20", rec.Collapsed.ReclaimedBytes) + } +} diff --git a/internal/store/dedup.go b/internal/store/dedup.go new file mode 100644 index 0000000..38db578 --- /dev/null +++ b/internal/store/dedup.go @@ -0,0 +1,191 @@ +// Content-hash duplicate detection: the store's "you already scratched this" +// hygiene primitive. +// +// Throwaway files breed byte-identical copies — the same log pasted three +// times, an agent re-capturing identical output on every run, the same snippet +// `sp new`'d across two projects. Each copy ages out separately and wastes +// footprint. dedup hashes every live scratch's content, groups by digest, and +// reports the clusters of identical scratches; `--collapse` (opt-in) moves the +// redundant copies to the morgue, always keeping the oldest as canonical. +// +// dedup is distinct from doctor (which reconciles index-vs-disk drift, not +// content equality) and reap (which is TTL-based). Like every other operation +// here it is morgue-first: the collapse path never hard-deletes, and the +// default path never moves anything at all. +package store + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "sort" + + "github.com/rwrife/scratchpatch/internal/index" +) + +// DupMember is one scratch within a duplicate cluster, carrying the metadata +// the report needs without exposing the whole index record. Canonical marks the +// member the store would keep (the oldest) so the render layer doesn't have to +// re-derive it. +type DupMember struct { + Scratch index.Scratch + Canonical bool +} + +// DupCluster is a group of live scratches whose content hashes to the same +// digest: two or more byte-identical copies. Members are ordered +// canonical-first (oldest), then newest-first for the redundant copies, so the +// report reads top-down as "keep this, these are the extras". WastedBytes is the +// footprint the redundant copies cost (every member past the canonical one), +// i.e. what `--collapse` would reclaim from the live set. +type DupCluster struct { + Digest string + Members []DupMember + WastedBytes int64 +} + +// Canonical returns the cluster's keep-member (the oldest). A well-formed +// cluster always has at least two members, so Members[0] is safe. +func (c DupCluster) Canonical() index.Scratch { return c.Members[0].Scratch } + +// Redundant returns the non-canonical members: the copies `--collapse` would +// move to the morgue. +func (c DupCluster) Redundant() []DupMember { return c.Members[1:] } + +// DedupReport is the full read-only result of scanning the live set for exact +// duplicates. Clusters is empty when the store is unique. TotalWasted is the +// sum of every cluster's WastedBytes — the bytes reclaimable by collapsing. +// ScannedCount is how many live scratches were hashed, so the report can say +// "scanned N, found M clusters". +type DedupReport struct { + Clusters []DupCluster + TotalWasted int64 + ScannedCount int +} + +// Clean reports whether no duplicates were found. +func (r DedupReport) Clean() bool { return len(r.Clusters) == 0 } + +// Dedup hashes the content of every live scratch and groups byte-identical +// copies into clusters. It is strictly read-only — nothing is moved or deleted. +// +// A scratch whose content file can't be read (an orphaned/missing entry doctor +// would flag) is skipped rather than aborting the scan, so one bad file doesn't +// hide every real duplicate; the skip is surfaced via the returned error only +// if reading the index itself fails. Content equality is by digest alone, so +// identical bytes cluster together regardless of differing names, tags, or ext. +func (s *Store) Dedup() (DedupReport, error) { + live, err := s.ListLive() + if err != nil { + return DedupReport{}, fmt.Errorf("dedup: list live: %w", err) + } + + // groups maps a content digest → the scratches that hashed to it, in + // index order (newest-first, per ListLive). + groups := make(map[string][]index.Scratch) + var order []string // first-seen digest order, for deterministic output + scanned := 0 + + for _, sc := range live { + b, err := os.ReadFile(s.LivePath(sc)) + if err != nil { + // Unreadable/missing content: skip it. doctor is the tool that + // reports such drift; dedup shouldn't crash on it. + continue + } + scanned++ + sum := sha256.Sum256(b) + digest := hex.EncodeToString(sum[:]) + if _, ok := groups[digest]; !ok { + order = append(order, digest) + } + groups[digest] = append(groups[digest], sc) + } + + var report DedupReport + report.ScannedCount = scanned + + for _, digest := range order { + members := groups[digest] + if len(members) < 2 { + continue // unique content is not a cluster + } + cluster := buildCluster(digest, members) + report.Clusters = append(report.Clusters, cluster) + report.TotalWasted += cluster.WastedBytes + } + + // Deterministic cluster order: most-wasteful first, ties broken by the + // canonical member's id so runs and tests are stable. + sort.SliceStable(report.Clusters, func(i, j int) bool { + if report.Clusters[i].WastedBytes != report.Clusters[j].WastedBytes { + return report.Clusters[i].WastedBytes > report.Clusters[j].WastedBytes + } + return report.Clusters[i].Canonical().ID < report.Clusters[j].Canonical().ID + }) + + return report, nil +} + +// buildCluster orders a set of same-digest scratches into a DupCluster: the +// oldest (by CreatedAt, ties broken by id) is the canonical keep-member, and the +// rest are the redundant copies ordered newest-first. WastedBytes sums the sizes +// of every redundant member. +func buildCluster(digest string, members []index.Scratch) DupCluster { + ordered := make([]index.Scratch, len(members)) + copy(ordered, members) + // Oldest first so ordered[0] is canonical. + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].CreatedAt.Equal(ordered[j].CreatedAt) { + return ordered[i].ID < ordered[j].ID + } + return ordered[i].CreatedAt.Before(ordered[j].CreatedAt) + }) + + cluster := DupCluster{Digest: digest} + for i, sc := range ordered { + canonical := i == 0 + cluster.Members = append(cluster.Members, DupMember{Scratch: sc, Canonical: canonical}) + if !canonical { + cluster.WastedBytes += sc.Size + } + } + return cluster +} + +// CollapseResult records what a collapse run moved to the morgue: the ids that +// were morgued and the bytes reclaimed from the live set. Kept as plain data so +// the render layer can report it without re-deriving anything. +type CollapseResult struct { + // MovedIDs are the redundant scratch ids that were moved to the morgue, + // in cluster/order. + MovedIDs []string + // ReclaimedBytes is the total size of the moved copies. + ReclaimedBytes int64 +} + +// Collapse moves the redundant copies in report's clusters to the morgue, +// keeping each cluster's canonical (oldest) member live. It never hard-deletes: +// collapsed copies land in the morgue exactly like `sp rm`, recoverable with +// `sp resurrect` until the reaper purges them past the grace window. +// +// Collapse re-reads nothing and trusts the report it's handed — callers pass the +// report from a preceding Dedup() so the "what will move" preview and the actual +// move agree. Each move goes through MoveToMorgue, so index/filesystem stay in +// lockstep; a failure on any member aborts with what moved so far reported via +// the returned CollapseResult (already-moved copies stay morgued and recoverable). +func (s *Store) Collapse(report DedupReport) (CollapseResult, error) { + var result CollapseResult + for _, cluster := range report.Clusters { + for _, m := range cluster.Redundant() { + moved, err := s.MoveToMorgue(m.Scratch) + if err != nil { + return result, fmt.Errorf("collapse %s: %w", m.Scratch.ID, err) + } + result.MovedIDs = append(result.MovedIDs, moved.ID) + result.ReclaimedBytes += m.Scratch.Size + } + } + return result, nil +} diff --git a/internal/store/dedup_test.go b/internal/store/dedup_test.go new file mode 100644 index 0000000..fe74421 --- /dev/null +++ b/internal/store/dedup_test.go @@ -0,0 +1,224 @@ +package store + +import ( + "os" + "testing" + "time" + + "github.com/rwrife/scratchpatch/internal/index" +) + +// seedAt is like seed but stamps a specific CreatedAt so tests can control +// which member of a duplicate cluster is the oldest (canonical). It writes the +// content, then rewrites the index record's CreatedAt. +func seedAt(t *testing.T, s *Store, name, body string, created time.Time) index.Scratch { + t.Helper() + sc := seed(t, s, name, body) + sc.CreatedAt = created + if err := s.Index().Put(sc); err != nil { + t.Fatalf("Put(%q): %v", name, err) + } + return sc +} + +// TestDedupNoDuplicates: a store where every scratch has unique content reports +// clean, with the scanned count set and no clusters. +func TestDedupNoDuplicates(t *testing.T) { + s, _ := OpenWith(testConfig(t)) + seed(t, s, "alpha", "one\n") + seed(t, s, "beta", "two\n") + seed(t, s, "gamma", "three\n") + + r, err := s.Dedup() + if err != nil { + t.Fatalf("Dedup: %v", err) + } + if !r.Clean() { + t.Errorf("expected clean, got clusters=%+v", r.Clusters) + } + if r.ScannedCount != 3 { + t.Errorf("scanned=%d, want 3", r.ScannedCount) + } + if r.TotalWasted != 0 { + t.Errorf("wasted=%d, want 0", r.TotalWasted) + } +} + +// TestDedupEmptyStore: an empty store is trivially clean. +func TestDedupEmptyStore(t *testing.T) { + s, _ := OpenWith(testConfig(t)) + r, err := s.Dedup() + if err != nil { + t.Fatalf("Dedup: %v", err) + } + if !r.Clean() || r.ScannedCount != 0 { + t.Errorf("empty store: clean=%v scanned=%d, want true/0", r.Clean(), r.ScannedCount) + } +} + +// TestDedupSingleScratch: one scratch can never be a duplicate of anything. +func TestDedupSingleScratch(t *testing.T) { + s, _ := OpenWith(testConfig(t)) + seed(t, s, "solo", "lonely\n") + r, err := s.Dedup() + if err != nil { + t.Fatalf("Dedup: %v", err) + } + if !r.Clean() { + t.Errorf("single scratch should be clean, got %+v", r.Clusters) + } +} + +// TestDedupExactDuplicates: identical content clusters together, the oldest is +// canonical, and wasted bytes count only the redundant copies. Different names +// with identical bytes are still a duplicate. +func TestDedupExactDuplicates(t *testing.T) { + s, _ := OpenWith(testConfig(t)) + base := time.Now().Add(-time.Hour) + body := "duplicate me\n" // 13 bytes + + old := seedAt(t, s, "original", body, base) // oldest → canonical + mid := seedAt(t, s, "copy-a", body, base.Add(10*time.Minute)) // redundant + newest := seedAt(t, s, "copy-b", body, base.Add(20*time.Minute)) // redundant + seed(t, s, "unique", "not a dup\n") // control + + r, err := s.Dedup() + if err != nil { + t.Fatalf("Dedup: %v", err) + } + if len(r.Clusters) != 1 { + t.Fatalf("clusters=%d, want 1: %+v", len(r.Clusters), r.Clusters) + } + c := r.Clusters[0] + if len(c.Members) != 3 { + t.Fatalf("members=%d, want 3", len(c.Members)) + } + if c.Canonical().ID != old.ID { + t.Errorf("canonical=%s, want oldest %s", c.Canonical().ID, old.ID) + } + if !c.Members[0].Canonical || c.Members[1].Canonical || c.Members[2].Canonical { + t.Errorf("only Members[0] should be canonical: %+v", c.Members) + } + // Two redundant copies of 13 bytes each = 26 wasted. + wantWasted := int64(len(body) * 2) + if c.WastedBytes != wantWasted { + t.Errorf("wasted=%d, want %d", c.WastedBytes, wantWasted) + } + if r.TotalWasted != wantWasted { + t.Errorf("total wasted=%d, want %d", r.TotalWasted, wantWasted) + } + // Redundant copies are the two non-oldest ids. + redundant := map[string]bool{} + for _, m := range c.Redundant() { + redundant[m.Scratch.ID] = true + } + if !redundant[mid.ID] || !redundant[newest.ID] { + t.Errorf("redundant set = %v, want %s and %s", redundant, mid.ID, newest.ID) + } +} + +// TestDedupSkipsUnreadableContent: a scratch whose content file is gone is +// skipped (not counted, not crashed), and the remaining real duplicates still +// cluster. +func TestDedupSkipsUnreadableContent(t *testing.T) { + s, _ := OpenWith(testConfig(t)) + body := "shared\n" + seedAt(t, s, "a", body, time.Now().Add(-time.Hour)) + seedAt(t, s, "b", body, time.Now().Add(-30*time.Minute)) + broken := seed(t, s, "broken", "orphaned\n") + + // Remove the content file out from under the index to simulate drift. + if err := os.Remove(s.LivePath(broken)); err != nil { + t.Fatalf("remove content: %v", err) + } + + r, err := s.Dedup() + if err != nil { + t.Fatalf("Dedup: %v", err) + } + if r.ScannedCount != 2 { + t.Errorf("scanned=%d, want 2 (broken skipped)", r.ScannedCount) + } + if len(r.Clusters) != 1 { + t.Fatalf("clusters=%d, want 1", len(r.Clusters)) + } +} + +// TestDedupReadOnly: the default Dedup path moves nothing — the live set is +// unchanged and nothing is morgued. +func TestDedupReadOnly(t *testing.T) { + s, _ := OpenWith(testConfig(t)) + body := "keep both live\n" + seedAt(t, s, "a", body, time.Now().Add(-time.Hour)) + seedAt(t, s, "b", body, time.Now().Add(-time.Minute)) + + if _, err := s.Dedup(); err != nil { + t.Fatalf("Dedup: %v", err) + } + + live, err := s.ListLive() + if err != nil { + t.Fatalf("ListLive: %v", err) + } + if len(live) != 2 { + t.Errorf("live=%d after read-only dedup, want 2", len(live)) + } + morgue, err := s.ListMorgue() + if err != nil { + t.Fatalf("ListMorgue: %v", err) + } + if len(morgue) != 0 { + t.Errorf("morgue=%d after read-only dedup, want 0", len(morgue)) + } +} + +// TestCollapseMovesRedundantToMorgue: --collapse morgues the redundant copies, +// keeps the canonical live, never hard-deletes, and reports what moved. +func TestCollapseMovesRedundantToMorgue(t *testing.T) { + s, _ := OpenWith(testConfig(t)) + base := time.Now().Add(-time.Hour) + body := "collapse me\n" // 12 bytes + + canonical := seedAt(t, s, "keep", body, base) + redA := seedAt(t, s, "dup-a", body, base.Add(time.Minute)) + redB := seedAt(t, s, "dup-b", body, base.Add(2*time.Minute)) + + report, err := s.Dedup() + if err != nil { + t.Fatalf("Dedup: %v", err) + } + res, err := s.Collapse(report) + if err != nil { + t.Fatalf("Collapse: %v", err) + } + + if len(res.MovedIDs) != 2 { + t.Fatalf("moved=%d, want 2: %v", len(res.MovedIDs), res.MovedIDs) + } + moved := map[string]bool{res.MovedIDs[0]: true, res.MovedIDs[1]: true} + if !moved[redA.ID] || !moved[redB.ID] { + t.Errorf("moved=%v, want %s and %s", res.MovedIDs, redA.ID, redB.ID) + } + if moved[canonical.ID] { + t.Errorf("canonical %s should not have moved", canonical.ID) + } + if res.ReclaimedBytes != int64(len(body)*2) { + t.Errorf("reclaimed=%d, want %d", res.ReclaimedBytes, len(body)*2) + } + + // Canonical stays live; redundant copies are morgued, not gone. + live, _ := s.ListLive() + if len(live) != 1 || live[0].ID != canonical.ID { + t.Errorf("live set = %+v, want just %s", live, canonical.ID) + } + morgue, _ := s.ListMorgue() + if len(morgue) != 2 { + t.Errorf("morgue=%d, want 2 (nothing hard-deleted)", len(morgue)) + } + // Content is recoverable: the morgued files still exist on disk. + for _, m := range morgue { + if _, err := os.Stat(s.LivePath(m)); err != nil { + t.Errorf("morgued content %s missing: %v", m.ID, err) + } + } +}