From d2d2d55942bcfd9c4abb11aa48b7c202ab002291 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Wed, 18 Nov 2026 17:39:27 -0300 Subject: [PATCH] mutate: a test that fails with nothing broken fails the run, and the copy answers git A test that failed in mutate's copy of the module was said and left out, and the run passed: a green check over a FAIL. It now asks every edit and then exits 2 naming the test. The two that failed so here: versions/gate's TestThisRepositoryPassesTheGate asks git which files the module holds, and the copy had no repository - it now is one of its own, every file it holds staged, never the checkout's .git; and the socket test's path, under mutate's own run, was 17 bytes deeper than 108 allow - it now names the short TMPDIR it holds the tree to. Co-Authored-By: Claude Opus 5.5 --- internal/mutate/runner.go | 19 ++++++++++++++ internal/mutate/runner_test.go | 9 ++++--- internal/mutate/tree.go | 37 ++++++++++++++++++++++----- internal/mutate/tree_internal_test.go | 33 ++++++++++++++++++++++++ site/src/content/docs/mutate.mdx | 17 +++++++++--- 5 files changed, 102 insertions(+), 13 deletions(-) diff --git a/internal/mutate/runner.go b/internal/mutate/runner.go index b036b11..073c48f 100644 --- a/internal/mutate/runner.go +++ b/internal/mutate/runner.go @@ -23,6 +23,8 @@ type Tally struct { // Wrote are the files the tests left, changed or removed where they ran, by their paths in // the source, which none of them was written to (Tree). Wrote []string + // Broken are the tests that fail with nothing broken, as ": ", sorted. + Broken []string } // result is what the tests did about one mutation. @@ -114,6 +116,23 @@ func RunAll(chosen []Mutation, importers map[string][]string, timeout string, jo _, _ = fmt.Fprintf(w, " the tests left or changed these in their copy of the module, and not in the source: %s\n", strings.Join(t.Wrote, ", ")) } + // Every package's coverage is measured by now: each run asked for it before its first edit. + for dir, c := range covers.by { + if c.c != nil { + for test := range c.c.failing { + t.Broken = append(t.Broken, dir+": "+test) + } + } + } + slices.Sort(t.Broken) + // After every edit was asked, so the run says all it can first. A test that fails with nothing + // broken is asked about no edit, so what it would have refused is let through, and the run + // that reports it and passes is a green check over a failing test. + if len(t.Broken) > 0 && firstErr == nil { + firstErr = fmt.Errorf("mutate: %d test(s) fail with nothing broken, so no edit was asked of them: %s. "+ + "Each failed in this run's copy of the module, as said above: fix it so that it passes there", + len(t.Broken), strings.Join(t.Broken, ", ")) + } return t, firstErr } diff --git a/internal/mutate/runner_test.go b/internal/mutate/runner_test.go index 6a858ff..22a8116 100644 --- a/internal/mutate/runner_test.go +++ b/internal/mutate/runner_test.go @@ -93,7 +93,8 @@ func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { // A test that fails with nothing broken refuses no edit: the coverage the run measures first says // so, for an edit in the package's binary and for one built alone. Asked anyway, it would refuse -// every edit, and a change would pass the gate on a broken test. +// every edit, and a change would pass the gate on a broken test. And the run fails naming it, once +// every edit was asked: passing, it is a green check over a test that fails. func TestATestThatAlwaysFailsRefusesNothing(t *testing.T) { dir := t.TempDir() for name, body := range map[string]string{ @@ -110,10 +111,12 @@ func TestATestThatAlwaysFailsRefusesNothing(t *testing.T) { } var said strings.Builder tally, err := mutate.RunAll(plan, nil, "2m", 2, &said) - require.NoError(t, err) + require.Error(t, err, "a run with a test that fails with nothing broken passed") + assert.Contains(t, err.Error(), dir+": TestBroken") + assert.Equal(t, []string{dir + ": TestBroken"}, tally.Broken) assert.Contains(t, said.String(), dir+": 1 edit(s) in one binary, 1 built alone", "One's literal in the binary, Wide's alone") assert.Zero(t, tally.Killed, "a test broken regardless refused an edit: %s", said.String()) - assert.Len(t, tally.Survived, 2, "%s", said.String()) + assert.Len(t, tally.Survived, 2, "every edit is still asked, and said: %s", said.String()) } // A package with no tests of its own is asked through the packages that import it, each built diff --git a/internal/mutate/tree.go b/internal/mutate/tree.go index 5cc04d6..dd6d3ec 100644 --- a/internal/mutate/tree.go +++ b/internal/mutate/tree.go @@ -24,7 +24,8 @@ import ( // The copy is the module as git sees it: its tracked files and those it does not ignore, so a // test reads testdata/ and ../../docs as it would in place. A directory git ignores (node_modules, // a build's output) and a repository nested in the module are linked, not copied: read as they -// are, and shared. Outside a git work tree it is every file under the module's root. The tests +// are, and shared. It is a git repository of its own, those files staged (copyModule says why). +// Outside a git work tree it is every file under the module's root. The tests // are built where the sources are, and only run here. type Tree struct { dir string @@ -148,7 +149,7 @@ type treeFile struct { } func copyModule(src, dst string) (*treeCopy, error) { - files, err := moduleFiles(src) + files, inGit, err := moduleFiles(src) if err != nil { return nil, err } @@ -161,16 +162,33 @@ func copyModule(src, dst string) (*treeCopy, error) { return nil, err } } + if !inGit { + return c, nil + } + // A repository of its own, every file the copy holds in its index: a test that asks git what + // the module holds - a gate over the files git lists - is answered as in the checkout. Without + // one it is told "not a git repository", fails with nothing broken, and refuses no edit. Never + // the checkout's own .git: a test's git that lands there writes into what every worktree + // commits from. + for _, args := range [][]string{{"init", "-q"}, {"add", "-A"}} { + git := exec.Command("git", args...) + git.Dir = dst + if _, err := output("listing", git); err != nil { + return nil, fmt.Errorf("git %s in the copy: %w", strings.Join(args, " "), err) + } + } return c, nil } // moduleFiles is what of the module under root is copied, by its path from root: what git -// tracks or does not ignore, and each directory it ignores whole, which is linked. -func moduleFiles(root string) ([]string, error) { +// tracks or does not ignore, and each directory it ignores whole, which is linked; and whether +// root is in a git work tree, which is what said so. +func moduleFiles(root string) ([]string, bool, error) { inside := exec.Command("git", "rev-parse", "--is-inside-work-tree") inside.Dir = root if _, err := output("listing", inside); err != nil { - return walkFiles(root) + files, err := walkFiles(root) + return files, false, err } var out []string for _, args := range [][]string{ @@ -181,7 +199,7 @@ func moduleFiles(root string) ([]string, error) { list.Dir = root said, err := output("listing", list) if err != nil { - return nil, fmt.Errorf("git %s: %w", strings.Join(args, " "), err) + return nil, false, fmt.Errorf("git %s: %w", strings.Join(args, " "), err) } ignored := slices.Contains(args, "--ignored") for path := range strings.SplitSeq(string(said), "\x00") { @@ -194,7 +212,7 @@ func moduleFiles(root string) ([]string, error) { out = append(out, filepath.FromSlash(strings.TrimSuffix(path, "/"))) } } - return out, nil + return out, true, nil } // walkFiles is every file under root but git's own. @@ -306,6 +324,11 @@ func (c *treeCopy) reset(wrote func(rel string)) error { if err != nil || rel == "." { return err } + if rel == ".git" { + // The copy's own repository, which no source file is: what a test does to it is + // not put back - one that needs a repository to change makes its own. + return fs.SkipDir + } if d.IsDir() { if c.dirs[rel] { return nil diff --git a/internal/mutate/tree_internal_test.go b/internal/mutate/tree_internal_test.go index 0b947bc..6661c25 100644 --- a/internal/mutate/tree_internal_test.go +++ b/internal/mutate/tree_internal_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" "github.com/spin-stack/go-tools/internal/gittest" @@ -67,6 +68,10 @@ func TestATreeIsTheModuleAsGitSeesItPutBackAfterEachRun(t *testing.T) { // A run's temporary directory is empty, its own and gone once it returns, and short enough that // a test with a long name binds a socket in its t.TempDir() there, however deep the tree is. func TestARunsTemporaryDirectoryTakesASocketFromATestWithALongName(t *testing.T) { + // The run's directory is made under TMPDIR, as short as a runner's here. Under mutate's own + // run TMPDIR is already a run's directory, 17 bytes deeper, and this socket's path would be + // over the 108 bytes one may have: a failure of where the test ran, not of what it holds. + t.Setenv("TMPDIR", "/tmp") root := gitModule(t) tree := NewTree(filepath.Join(t.TempDir(), "trees", "12", "a-tree-as-deep-as-a-runner-makes-it")) var tmps []string @@ -255,3 +260,31 @@ func putBack(t *testing.T, root, dir string) { } } } + +// A test that asks git what the module holds is answered in the copy as in the checkout: the copy +// is a repository of its own with every file it holds staged, which no run's reset takes away or +// names as written. Without one, a gate over the files git lists fails with nothing broken. +func TestATestThatAsksGitWhatTheModuleHoldsIsAnsweredInTheCopy(t *testing.T) { + root := gitModule(t) + tree := NewTree(t.TempDir()) + listing := []string{"ls-files", "--cached", "--others", "--exclude-standard"} + // Sorted: the checkout lists what it tracks before what it does not, and the copy tracks all. + sorted := func(listed string) string { + lines := strings.Split(listed, "\n") + slices.Sort(lines) + return strings.Join(lines, "\n") + } + want := sorted(gittest.Run(t, filepath.Join(root, "p"), listing...)) + for range 2 { + if err := tree.in(filepath.Join(root, "p"), func(at, _ string) { + if got := sorted(gittest.Run(t, at, listing...)); got != want { + t.Errorf("git in the copy lists\n%s\nnot the checkout's\n%s", got, want) + } + }); err != nil { + t.Fatal(err) + } + } + if got := tree.Wrote(); len(got) != 0 { + t.Errorf("the copy's own repository was named as written by a run: %v", got) + } +} diff --git a/site/src/content/docs/mutate.mdx b/site/src/content/docs/mutate.mdx index 204c049..5fd8a09 100644 --- a/site/src/content/docs/mutate.mdx +++ b/site/src/content/docs/mutate.mdx @@ -157,7 +157,16 @@ cache that keeps a pull request's refused edits for its next run. | `-cache` | none | a directory to keep refused edits in between runs | | `-stale-exempts` | | report and fail on `mutate-exempt` reasons over nothing | -It exits 0 when every edit was refused, 1 when one survived, and 2 when it could not run. +It exits 0 when every edit was refused, 1 when one survived, and 2 when it could not run or a test +failed with nothing broken. + +### A test that fails with nothing broken + +Before it breaks anything, `mutate` runs each test once in its copy of the module. A test that +fails there can't refuse an edit, since it fails whatever the edit, so no edit is asked of it. The +run still asks every edit and prints the test's output, then exits 2 naming the package and the +test. A pass here would be a green check over a failing test. The fix is the test's: what it reads +that the copy doesn't have, or a path it shares with a test run beside it. ### Tests that need an environment @@ -183,8 +192,10 @@ the copy: a test with a long name that binds a unix socket in its `t.TempDir()` The copy holds what git sees: tracked files and files git doesn't ignore. Directories git ignores (`node_modules`, a build's output) and repositories nested in the module are linked rather than -copied, so tests can read them, but those are shared. Outside a git work tree, the copy is every -file under the module's root. Packages are still built from your checkout, and only their tests run +copied, so tests can read them, but those are shared. The copy is a git repository of its own with +every file it holds staged, never your checkout's `.git`: a test that asks git which files the +module has gets the same answer as in your checkout. Outside a git work tree, the copy is every +file under the module's root, and no repository. Packages are still built from your checkout, and only their tests run in the copy. When the tests leave files behind, or change or remove one, the run's report lists them. Those