sp dedup — content-hash duplicate detection (#28)

README.md

@@ -26,6 +26,8 @@ sp doctor --json | jq -e '.healthy'  # gate a script on store health
 sp ls --json | jq '.[].id'           # machine-readable output for scripting
 sp completion zsh > "${fpath[1]}/_sp" # tab-completion for your shell
 sp scan <id>                         # tripwire: does this scratch hold a secret?
+sp dedup                             # report byte-identical scratches (read-only)
+sp dedup --collapse                  # send redundant copies to the morgue (morgue-first)
 sp resurrect <id>                    # changed your mind? pull it back
 sp promote <id>                      # the good ones: graduate a scratch into your repo
 ```
@@ -40,7 +42,9 @@ scripting polish — **`sp ls --json`** / **`sp doctor --json`** and
 **`sp completion`** for bash/zsh/fish; **`sp promote`** to graduate a scratch
 into your repo; and a **secret tripwire** — **`sp scan`** flags scratches that
 look like they hold credentials, `sp ls` marks them with a 🔑, and `sp promote`
-refuses them unless you pass `--allow-secrets` (M6, in progress).
+refuses them unless you pass `--allow-secrets`; and **content dedup** —
+**`sp dedup`** reports byte-identical scratches and `--collapse` sweeps the
+redundant copies to the morgue (M6, in progress).
 
 ### `sp new [name]`
 
@@ -319,6 +323,39 @@ The tripwire also shows up where it matters most:
   you pass `--allow-secrets` — the last line of defense before a leaked key
   lands somewhere it might get committed.
 
+### `sp dedup` — find (and collapse) byte-identical scratches
+
+Throwaway files breed duplicates: the same log pasted three times, an agent
+re-capturing identical output on every run, the same snippet `sp new`'d across
+two projects. Each copy ages out separately and quietly wastes footprint.
+`sp dedup` hashes every **live** scratch's content (SHA-256), groups the
+byte-identical copies into clusters, and reports how much they cost — naming the
+oldest copy in each cluster as the **canonical** one to keep.
+
+This is a content-equality primitive, distinct from `sp doctor` (which
+reconciles index-vs-disk drift, not equality) and `sp reap` (TTL-based). Two
+scratches with different names, tags, or extensions but identical bytes are
+still duplicates.
+
+```bash
+sp dedup             # read-only report of duplicate clusters (canonical + wasted bytes)
+sp dedup --no-color  # plain, script-friendly
+sp dedup --json      # stable JSON object (no color, no flavor)
+sp dedup --collapse  # move the redundant copies to the morgue, keep the canonical
+```
+
+By default `sp dedup` is **strictly read-only** — it moves nothing. Pass
+`--collapse` to send the redundant copies to the morgue, keeping each cluster's
+canonical (oldest) member live. Like `sp rm`, collapse is **morgue-first and
+never hard-deletes**: collapsed copies are recoverable with `sp resurrect` until
+the reaper purges them past the grace window.
+
+A unique store prints a clean, one-line bill of health. The `--json` form
+carries a top-level `clean` flag plus `clusters` (always an array, never null;
+each cluster has its full `digest`, `wastedBytes`, and `members` with a
+`canonical` flag) and a `collapsed` object that is `null` until a `--collapse`
+run moves something: `sp dedup --json | jq -e '.clean'`.
+
 ### `sp completion <bash|zsh|fish>`
 
 Prints a shell completion script to stdout so `sp`'s commands and flags

internal/cli/dedup.go

@@ -0,0 +1,114 @@
+package cli
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/rwrife/scratchpatch/internal/render"
+	"github.com/rwrife/scratchpatch/internal/store"
+)
+
+func newDedupCommand() *cobra.Command {
+	var noColor bool
+	var asJSON bool
+	var collapse bool
+
+	cmd := &cobra.Command{
+		Use:   "dedup",
+		Short: "Find (and optionally collapse) byte-identical scratches",
+		Long: "Throwaway files breed duplicates: the same log pasted three times, an\n" +
+			"agent re-capturing identical output on every run, the same snippet\n" +
+			"`sp new`'d across two projects. dedup hashes every live scratch's content\n" +
+			"and groups the byte-identical copies into clusters.\n\n" +
+			"By default dedup is strictly read-only: it reports the clusters, names the\n" +
+			"oldest copy as canonical, and shows how many bytes the redundant copies\n" +
+			"waste — it moves nothing.\n\n" +
+			"Pass --collapse to send the redundant copies to the morgue, keeping each\n" +
+			"cluster's canonical (oldest) member live. Like `sp rm`, this never\n" +
+			"hard-deletes: collapsed copies are recoverable with `sp resurrect` until\n" +
+			"the reaper purges them past the grace window.\n\n" +
+			"dedup is content-equality only — distinct from `sp doctor` (index-vs-disk\n" +
+			"drift) and `sp reap` (TTL-based). Pass --json for a stable, machine-\n" +
+			"readable object (no color, no flavor): `sp dedup --json | jq '.clean'`.",
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runDedup(cmd, noColor, asJSON, collapse)
+		},
+	}
+
+	cmd.Flags().BoolVar(&noColor, "no-color", false, "force plain output even on a TTY")
+	cmd.Flags().BoolVar(&asJSON, "json", false, "emit a JSON object instead of a report (for scripting)")
+	cmd.Flags().BoolVar(&collapse, "collapse", false, "move redundant copies to the morgue, keeping the oldest as canonical")
+
+	return cmd
+}
+
+func runDedup(cmd *cobra.Command, noColor, asJSON, collapse bool) error {
+	st, err := store.Open()
+	if err != nil {
+		return err
+	}
+
+	report, err := st.Dedup()
+	if err != nil {
+		return err
+	}
+
+	data := toDedupData(report)
+
+	// --collapse moves the redundant copies to the morgue, then records the
+	// outcome on the report so both the human and JSON views can confirm it.
+	if collapse && !report.Clean() {
+		res, err := st.Collapse(report)
+		if err != nil {
+			return err
+		}
+		data.Collapsed = &render.DedupCollapsedData{
+			MovedIDs:       res.MovedIDs,
+			ReclaimedBytes: res.ReclaimedBytes,
+		}
+	} else if collapse {
+		// Nothing to collapse, but the user asked — report an empty outcome so
+		// the message is "collapsed 0" rather than the read-only footer.
+		data.Collapsed = &render.DedupCollapsedData{}
+	}
+
+	out := cmd.OutOrStdout()
+
+	// --json is intentionally color-free and personality-free; never tint it,
+	// regardless of TTY or --no-color, matching `sp ls --json`/`sp doctor --json`.
+	if asJSON {
+		return render.DedupReportJSON(out, data)
+	}
+
+	color := !noColor && isTerminal(out)
+	return render.DedupReport(out, data, color)
+}
+
+// toDedupData flattens the store's DedupReport into the render layer's plain
+// view, so render never has to import the store package — the same adapter
+// pattern doctor.go/reap.go use.
+func toDedupData(r store.DedupReport) render.DedupData {
+	clusters := make([]render.DedupClusterData, 0, len(r.Clusters))
+	for _, c := range r.Clusters {
+		members := make([]render.DedupMemberData, 0, len(c.Members))
+		for _, m := range c.Members {
+			members = append(members, render.DedupMemberData{
+				ID:        m.Scratch.ID,
+				Name:      m.Scratch.Name,
+				Size:      m.Scratch.Size,
+				CreatedAt: m.Scratch.CreatedAt,
+				Canonical: m.Canonical,
+			})
+		}
+		clusters = append(clusters, render.DedupClusterData{
+			Digest:      c.Digest,
+			Members:     members,
+			WastedBytes: c.WastedBytes,
+		})
+	}
+	return render.DedupData{
+		Clusters:     clusters,
+		TotalWasted:  r.TotalWasted,
+		ScannedCount: r.ScannedCount,
+	}
+}

internal/cli/dedup_test.go

@@ -0,0 +1,132 @@
+package cli
+
+import (
+	"encoding/json"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// TestDedupCleanStoreCLI: a store of unique scratches reports no duplicates.
+func TestDedupCleanStoreCLI(t *testing.T) {
+	s := newSession(t)
+	if out, err := s.run("new", "a", "--content", "alpha\n", "--ext", "txt"); err != nil {
+		t.Fatalf("new a: %v (%s)", err, out)
+	}
+	if out, err := s.run("new", "b", "--content", "beta\n", "--ext", "txt"); err != nil {
+		t.Fatalf("new b: %v (%s)", err, out)
+	}
+
+	out, err := s.run("dedup", "--no-color")
+	if err != nil {
+		t.Fatalf("dedup: %v (%s)", err, out)
+	}
+	if !strings.Contains(out, "no duplicates") {
+		t.Errorf("clean store should report no duplicates; got %q", out)
+	}
+}
+
+// TestDedupReportsAndCollapsesCLI: identical content is reported as a cluster;
+// default is read-only; --collapse morgues the redundant copy (never hard-
+// deletes), keeping the canonical live.
+func TestDedupReportsAndCollapsesCLI(t *testing.T) {
+	s := newSession(t)
+	body := "duplicate body\n"
+	for _, name := range []string{"first", "second", "third"} {
+		if out, err := s.run("new", name, "--content", body, "--ext", "txt"); err != nil {
+			t.Fatalf("new %s: %v (%s)", name, err, out)
+		}
+	}
+
+	// Read-only report first.
+	out, err := s.run("dedup", "--no-color")
+	if err != nil {
+		t.Fatalf("dedup: %v (%s)", err, out)
+	}
+	if !strings.Contains(out, "cluster") || !strings.Contains(out, "canonical") {
+		t.Errorf("dedup should report a cluster with a canonical; got %q", out)
+	}
+	if !strings.Contains(out, "--collapse") {
+		t.Errorf("read-only dedup should point at --collapse; got %q", out)
+	}
+	// Read-only: still 3 live scratch files on disk.
+	if got := len(globTxt(t, s.home)); got != 3 {
+		t.Fatalf("read-only dedup changed the store: %d live files, want 3", got)
+	}
+
+	// Now collapse.
+	out, err = s.run("dedup", "--collapse", "--no-color")
+	if err != nil {
+		t.Fatalf("dedup --collapse: %v (%s)", err, out)
+	}
+	if !strings.Contains(out, "collapsed") || !strings.Contains(out, "morgue") {
+		t.Errorf("collapse should confirm the move; got %q", out)
+	}
+
+	// Canonical stays live; the two redundant copies are morgued (not gone).
+	live := globTxt(t, filepath.Join(s.home, "scratches"))
+	if len(live) != 1 {
+		t.Errorf("after collapse: %d live files, want 1", len(live))
+	}
+	morgue := globTxt(t, filepath.Join(s.home, "morgue"))
+	if len(morgue) != 2 {
+		t.Errorf("after collapse: %d morgue files, want 2 (nothing hard-deleted)", len(morgue))
+	}
+}
+
+// TestDedupJSONCLI: --json is a stable, colorless object with the clean flag
+// and cluster data.
+func TestDedupJSONCLI(t *testing.T) {
+	s := newSession(t)
+	body := "same\n"
+	for _, name := range []string{"x", "y"} {
+		if out, err := s.run("new", name, "--content", body, "--ext", "txt"); err != nil {
+			t.Fatalf("new %s: %v (%s)", name, err, out)
+		}
+	}
+
+	out, err := s.run("dedup", "--json")
+	if err != nil {
+		t.Fatalf("dedup --json: %v (%s)", err, out)
+	}
+	if strings.Contains(out, "\x1b[") {
+		t.Errorf("dedup --json must be colorless; got %q", out)
+	}
+
+	var rec struct {
+		Clean        bool `json:"clean"`
+		ClusterCount int  `json:"clusterCount"`
+		Clusters     []struct {
+			Count   int `json:"count"`
+			Members []struct {
+				Canonical bool `json:"canonical"`
+			} `json:"members"`
+		} `json:"clusters"`
+		Collapsed any `json:"collapsed"`
+	}
+	if err := json.Unmarshal([]byte(out), &rec); err != nil {
+		t.Fatalf("invalid JSON: %v (%q)", err, out)
+	}
+	if rec.Clean {
+		t.Errorf("two identical scratches should report clean=false")
+	}
+	if rec.ClusterCount != 1 || len(rec.Clusters) != 1 || rec.Clusters[0].Count != 2 {
+		t.Fatalf("unexpected cluster shape: %+v", rec)
+	}
+	if rec.Collapsed != nil {
+		t.Errorf("collapsed should be null without --collapse; got %v", rec.Collapsed)
+	}
+}
+
+// globTxt returns the *.txt files under dir (or under dir/scratches when dir is
+// a store home passed bare).
+func globTxt(t *testing.T, dir string) []string {
+	t.Helper()
+	// If dir is a store home (contains scratches/), glob live scratches.
+	pattern := filepath.Join(dir, "*.txt")
+	matches, _ := filepath.Glob(pattern)
+	if len(matches) == 0 {
+		matches, _ = filepath.Glob(filepath.Join(dir, "scratches", "*.txt"))
+	}
+	return matches
+}

internal/cli/root.go

@@ -54,6 +54,7 @@ func NewRootCommand() *cobra.Command {
 		newPromoteCommand(),
 		newReapCommand(),
 		newDoctorCommand(),
+		newDedupCommand(),
 		newScanCommand(),
 		newCompletionCommand(),
 	)