Wrap the test helper of every configured test path

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `test_paths:` in `.mutare.exs` names a single app's test directories, matching
+  `test_paths:` in its `mix.exs`. Mutare wraps each one's `test_helper.exs` with the
+  selector and coverage bootstrap, so a project whose tests live outside `test/` gets
+  coverage-based test selection instead of running the whole suite for every mutant. It
+  defaults to `["test"]`, the previous behaviour.
+
 ### Changed
 
 - OperandSwap no longer produces a swap that the expression around it cannot observe:

lib/mix/tasks/mutare.ex

@@ -359,6 +359,10 @@ defmodule Mix.Tasks.Mutare do
         expand_uses: true,
 
         # --- how the suite runs ---
+        # a single app's test directories, matching `test_paths:` in its mix.exs; each
+        # one's test_helper.exs gets the selector and coverage bootstrap. Umbrella apps
+        # always use their own `test/`
+        test_paths: ["test"],
         # :tests runs only the individual test cases covering each mutant; :coverage
         # runs whole covering files (opt-out for stateful async:false suites); :full
         # runs the whole suite for every mutant

lib/mutare/options.ex

@@ -33,6 +33,7 @@ defmodule Mutare.Options do
             MapSet.t({String.t(), pos_integer()} | {String.t(), pos_integer(), pos_integer()})
             | nil,
           test_selection: :tests | :coverage | :full,
+          test_paths: [String.t()],
           workers: pos_integer(),
           schedulers: pos_integer() | :all,
           partition_env: String.t() | nil,

lib/mutare/options/registry.ex

@@ -59,6 +59,14 @@ defmodule Mutare.Options.Registry do
         ":paths must be a non-empty list of path strings"
       )
 
+  defp validate_test_paths!(test_paths),
+    do:
+      validate!(
+        test_paths,
+        fn p -> is_list(p) and p != [] and Enum.all?(p, &is_binary/1) end,
+        ":test_paths must be a non-empty list of path strings"
+      )
+
   defp validate_exclude!(value),
     do:
       validate!(
@@ -722,6 +730,7 @@ defmodule Mutare.Options.Registry do
         validate: &validate_only_lines!/1
       ),
       spec(key: :test_selection, default: :tests, validate: &validate_test_selection!/1),
+      spec(key: :test_paths, default: ["test"], validate: &validate_test_paths!/1),
       spec(key: :workers, default: nil, cli: :integer, validate: &validate_workers!/1),
       spec(
         key: :schedulers,

lib/mutare/sandbox.ex

@@ -3,9 +3,10 @@ defmodule Mutare.Sandbox do
   Materialise a schema as a runnable copy of the target project.
 
   We copy the project to a dedicated directory (excluding build output), write
-  each metamutant over its original, and inject a tiny bootstrap into
-  `test/test_helper.exs` that combines `MUTARE_MUTANT_NAMESPACE` (the active
-  mutant's file) and `MUTARE_ACTIVE_MUTANT` (its id within that file) into one
+  each metamutant over its original, and inject a tiny bootstrap into each test
+  helper (`test/test_helper.exs`, or one per `:test_paths` entry) that combines
+  `MUTARE_MUTANT_NAMESPACE` (the active mutant's file) and
+  `MUTARE_ACTIVE_MUTANT` (its id within that file) into one
   `:persistent_term` entry before the suite starts. After the user's helper, Mutare appends a
   formatter that records a failed test (`Mutare.Sandbox.Command.FailureWitness`), so a run
   that crashes after a failure still counts as a kill. A second injection prefixes
@@ -149,10 +150,11 @@ defmodule Mutare.Sandbox do
   # ---------------------------------------------------------------------------
   """
 
-  # Stable relative paths for the two files Mutare generates (vs. copies). The
+  # Stable names for the two files Mutare generates (vs. copies): the test helper's
+  # basename, joined to each test path, and the coverage helper's relative path. The
   # coverage helper's collision-avoiding `_N` suffix is a fresh-mode-only fallback;
   # `keep_sandbox` materialisation always reuses this base path.
-  @helper_rel "test/test_helper.exs"
+  @helper_name "test_helper.exs"
   @default_helper "ExUnit.start()\n"
   @coverage_helper_rel "lib/__mutare__/coverage_helper.ex"
 
@@ -410,21 +412,24 @@ defmodule Mutare.Sandbox do
   end
 
   # The test helpers whose suite the runner will drive — each gets the selector/timeout/coverage
-  # bootstrap wrapped in by `helper_files/2`. A single project has one (`test/test_helper.exs`);
-  # an umbrella runs each app's suite sequentially in one BEAM (cwd = the app dir), so each app
+  # bootstrap wrapped in by `helper_files/3`. A single project has one per configured
+  # `:test_paths` entry (`test/test_helper.exs` by default); an umbrella runs each app's suite
+  # sequentially in one BEAM (cwd = the app dir), so each app
   # with a `test/` tree gets its own copy — and *every* app, not just the mutated ones, because a
   # mutant in one app can be killed by a test in another, and the selector must be live in
   # whichever app's process runs the line.
-  defp helper_rels(root, %{umbrella?: true, apps: apps}) do
+  defp helper_rels(root, %{umbrella?: true, apps: apps}, _test_paths) do
     for %{dir: dir} <- apps,
         app_dir = Path.join(root, dir),
         File.dir?(Path.join(app_dir, "test")),
         do: Path.join([dir, "test", "test_helper.exs"])
   end
 
-  # A single app (no project, or a non-umbrella one) keeps the pre-umbrella
-  # behavior: the root helper, created if absent.
-  defp helper_rels(_root, _project), do: [@helper_rel]
+  # A single app (no project, or a non-umbrella one): `mix test` requires the helper of every
+  # entry in the project's `:test_paths`, and Mutare reads none of the target's Mix config, so the
+  # paths come from the option. Each helper is created if absent.
+  defp helper_rels(_root, _project, test_paths),
+    do: Enum.map(test_paths, &Path.join(&1, @helper_name))
 
   # Wrap the user's test helper with the selector/timeout bootstrap, the coverage
   # probe (split around `ExUnit.start/0`; see the constants above) and the failure
@@ -475,7 +480,7 @@ defmodule Mutare.Sandbox do
       schema
       |> Schema.materialized_sources()
       |> Map.merge(coverage_helper_files(root, project))
-      |> Map.merge(helper_files(root, project, attribution_source(options)))
+      |> Map.merge(helper_files(root, project, options.test_paths, attribution_source(options)))
       |> Map.merge(config_files(root))
       |> Map.merge(RuntimeConfig.files(root, @excluded))
       |> Map.merge(project_files(root, project))
@@ -583,8 +588,8 @@ defmodule Mutare.Sandbox do
     )
   end
 
-  defp helper_files(root, project, attribution) do
-    Map.new(helper_rels(root, project), fn rel ->
+  defp helper_files(root, project, test_paths, attribution) do
+    Map.new(helper_rels(root, project, test_paths), fn rel ->
       user_helper =
         case File.read(Path.join(root, rel)) do
           {:ok, source} -> source

test/mutare/coverage_runner_test.exs

@@ -208,6 +208,55 @@ defmodule Mutare.CoverageRunnerTest do
     })
   end
 
+  describe "custom :test_paths (end to end)" do
+    test "the probe records through every configured test path's helper" do
+      %{project: project, sandbox: sandbox} =
+        Project.build(:custom_test_paths, %{
+          "mix.exs" => """
+          defmodule CustomTestPaths.MixProject do
+            use Mix.Project
+
+            def project,
+              do: [
+                app: :custom_test_paths,
+                version: "0.1.0",
+                elixir: "~> 1.15",
+                test_paths: ["apps/a/test", "apps/b/test"]
+              ]
+          end
+          """,
+          "lib/calc.ex" => "defmodule Calc do\n  def add(a, b), do: a + b\nend\n",
+          "apps/a/test/test_helper.exs" => "ExUnit.start()\n",
+          "apps/a/test/calc_test.exs" => """
+          defmodule CalcTest do
+            use ExUnit.Case
+            test "covers add loosely", do: assert(is_integer(Calc.add(2, 3)))
+          end
+          """,
+          "apps/b/test/test_helper.exs" => "ExUnit.start()\n",
+          "apps/b/test/other_test.exs" => """
+          defmodule OtherTest do
+            use ExUnit.Case
+            test "unrelated", do: assert(1 == 1)
+          end
+          """
+        })
+
+      File.rm_rf!(Path.join(project, "test"))
+
+      assert {:ok, run} =
+               Mutare.run(project,
+                 sandbox: sandbox,
+                 mutators: @probe,
+                 test_paths: ["apps/a/test", "apps/b/test"]
+               )
+
+      assert [%Result{status: :survived} = result] = run.results
+      assert result.selection == :tests
+      assert tests_run(result.output) == 1
+    end
+  end
+
   describe "test-case selection (:tests default, end to end)" do
     test "the default narrows to the covering test case within a multi-test file" do
       %{project: project, sandbox: sandbox} = two_test_project(:tests_narrow)

test/mutare/options/registry_test.exs

@@ -97,6 +97,7 @@ defmodule Mutare.Options.RegistryTest do
              {"only_files", "(all discovered files)"},
              {"only_lines", "(all lines)"},
              {"test_selection", "tests"},
+             {"test_paths", "[\"test\"]"},
              {"workers", to_string(System.schedulers_online() |> div(2) |> min(4) |> max(1))},
              {"schedulers", "#{Options.new([]).schedulers} per worker"},
              {"partition_env", "(off)"},

test/mutare/sandbox_test.exs

@@ -298,6 +298,34 @@ defmodule Mutare.SandboxTest do
     refute File.read!(Path.join(plain, "test/test_helper.exs")) =~ "coverage attribution"
   end
 
+  test "wraps the helper of every configured test path in a single app", context do
+    for dir <- ["apps/a/test", "apps/b/test"] do
+      File.mkdir_p!(Path.join(context.project, dir))
+
+      File.write!(
+        Path.join([context.project, dir, "test_helper.exs"]),
+        "UserHelper.#{Path.basename(Path.dirname(dir))}()\n"
+      )
+    end
+
+    sandbox = Path.join(context.base, "sandbox")
+
+    Sandbox.prepare(context.project, context.schema,
+      sandbox: sandbox,
+      test_paths: ["apps/a/test", "apps/b/test"]
+    )
+
+    for {dir, call} <- [{"apps/a/test", "UserHelper.a()"}, {"apps/b/test", "UserHelper.b()"}] do
+      helper = File.read!(Path.join([sandbox, dir, "test_helper.exs"]))
+      assert helper =~ "injected by Mutare: coverage dump"
+      [{user, _}] = :binary.matches(helper, call)
+      [{dump, _}] = :binary.matches(helper, "injected by Mutare: coverage dump")
+      assert user < dump
+    end
+
+    refute File.exists?(Path.join(sandbox, "test/test_helper.exs"))
+  end
+
   test "steps the generated support app aside from a real app at its name" do
     %{umbrella: umbrella, sandbox: sandbox} =
       Umbrella.build(:support_name_taken, %{

usage-rules/skills/mutare/references/troubleshooting.md

@@ -124,6 +124,10 @@ suppressing the mutant.
 
 ## Slow runs
 
+- **Every mutant runs the whole suite, and the probe reports `coverage dump unreadable
+  (:enoent)`.** The project sets `test_paths:` in its `mix.exs`, so `mix test` never loads
+  the `test/test_helper.exs` Mutare wrapped and the probe records nothing. Set the same
+  directories as `test_paths:` in `.mutare.exs`.
 - **Whole-suite fallbacks.** A mutant whose code runs in a process no test can be
   traced to runs the whole suite. After the coverage probe, a `↺` line says how many
   mutants will. `--verbose` tags each one's line `(whole suite)`, and the JSON marks