diff --git a/CHANGELOG.md b/CHANGELOG.md index 86f68523..115dabed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `test_paths:` in `.mutare.exs` names a single app's test directories, matching + `test_paths:` in its `mix.exs`. Mutare wraps each one's `test_helper.exs` with the + selector and coverage bootstrap, so a project whose tests live outside `test/` gets + coverage-based test selection instead of running the whole suite for every mutant. It + defaults to `["test"]`, the previous behaviour. + ### Changed - OperandSwap no longer produces a swap that the expression around it cannot observe: diff --git a/lib/mix/tasks/mutare.ex b/lib/mix/tasks/mutare.ex index dbcb5837..b63e45f2 100644 --- a/lib/mix/tasks/mutare.ex +++ b/lib/mix/tasks/mutare.ex @@ -359,6 +359,10 @@ defmodule Mix.Tasks.Mutare do expand_uses: true, # --- how the suite runs --- + # a single app's test directories, matching `test_paths:` in its mix.exs; each + # one's test_helper.exs gets the selector and coverage bootstrap. Umbrella apps + # always use their own `test/` + test_paths: ["test"], # :tests runs only the individual test cases covering each mutant; :coverage # runs whole covering files (opt-out for stateful async:false suites); :full # runs the whole suite for every mutant diff --git a/lib/mutare/options.ex b/lib/mutare/options.ex index eb7ed8dc..a2921f0f 100644 --- a/lib/mutare/options.ex +++ b/lib/mutare/options.ex @@ -33,6 +33,7 @@ defmodule Mutare.Options do MapSet.t({String.t(), pos_integer()} | {String.t(), pos_integer(), pos_integer()}) | nil, test_selection: :tests | :coverage | :full, + test_paths: [String.t()], workers: pos_integer(), schedulers: pos_integer() | :all, partition_env: String.t() | nil, diff --git a/lib/mutare/options/registry.ex b/lib/mutare/options/registry.ex index ac26de9d..5e28a84b 100644 --- a/lib/mutare/options/registry.ex +++ b/lib/mutare/options/registry.ex @@ -59,6 +59,14 @@ defmodule Mutare.Options.Registry do ":paths must be a non-empty list of path strings" ) + defp validate_test_paths!(test_paths), + do: + validate!( + test_paths, + fn p -> is_list(p) and p != [] and Enum.all?(p, &is_binary/1) end, + ":test_paths must be a non-empty list of path strings" + ) + defp validate_exclude!(value), do: validate!( @@ -722,6 +730,7 @@ defmodule Mutare.Options.Registry do validate: &validate_only_lines!/1 ), spec(key: :test_selection, default: :tests, validate: &validate_test_selection!/1), + spec(key: :test_paths, default: ["test"], validate: &validate_test_paths!/1), spec(key: :workers, default: nil, cli: :integer, validate: &validate_workers!/1), spec( key: :schedulers, diff --git a/lib/mutare/sandbox.ex b/lib/mutare/sandbox.ex index ee3cad2e..9ba25cc0 100644 --- a/lib/mutare/sandbox.ex +++ b/lib/mutare/sandbox.ex @@ -3,9 +3,10 @@ defmodule Mutare.Sandbox do Materialise a schema as a runnable copy of the target project. We copy the project to a dedicated directory (excluding build output), write - each metamutant over its original, and inject a tiny bootstrap into - `test/test_helper.exs` that combines `MUTARE_MUTANT_NAMESPACE` (the active - mutant's file) and `MUTARE_ACTIVE_MUTANT` (its id within that file) into one + each metamutant over its original, and inject a tiny bootstrap into each test + helper (`test/test_helper.exs`, or one per `:test_paths` entry) that combines + `MUTARE_MUTANT_NAMESPACE` (the active mutant's file) and + `MUTARE_ACTIVE_MUTANT` (its id within that file) into one `:persistent_term` entry before the suite starts. After the user's helper, Mutare appends a formatter that records a failed test (`Mutare.Sandbox.Command.FailureWitness`), so a run that crashes after a failure still counts as a kill. A second injection prefixes @@ -149,10 +150,11 @@ defmodule Mutare.Sandbox do # --------------------------------------------------------------------------- """ - # Stable relative paths for the two files Mutare generates (vs. copies). The + # Stable names for the two files Mutare generates (vs. copies): the test helper's + # basename, joined to each test path, and the coverage helper's relative path. The # coverage helper's collision-avoiding `_N` suffix is a fresh-mode-only fallback; # `keep_sandbox` materialisation always reuses this base path. - @helper_rel "test/test_helper.exs" + @helper_name "test_helper.exs" @default_helper "ExUnit.start()\n" @coverage_helper_rel "lib/__mutare__/coverage_helper.ex" @@ -410,21 +412,24 @@ defmodule Mutare.Sandbox do end # The test helpers whose suite the runner will drive — each gets the selector/timeout/coverage - # bootstrap wrapped in by `helper_files/2`. A single project has one (`test/test_helper.exs`); - # an umbrella runs each app's suite sequentially in one BEAM (cwd = the app dir), so each app + # bootstrap wrapped in by `helper_files/3`. A single project has one per configured + # `:test_paths` entry (`test/test_helper.exs` by default); an umbrella runs each app's suite + # sequentially in one BEAM (cwd = the app dir), so each app # with a `test/` tree gets its own copy — and *every* app, not just the mutated ones, because a # mutant in one app can be killed by a test in another, and the selector must be live in # whichever app's process runs the line. - defp helper_rels(root, %{umbrella?: true, apps: apps}) do + defp helper_rels(root, %{umbrella?: true, apps: apps}, _test_paths) do for %{dir: dir} <- apps, app_dir = Path.join(root, dir), File.dir?(Path.join(app_dir, "test")), do: Path.join([dir, "test", "test_helper.exs"]) end - # A single app (no project, or a non-umbrella one) keeps the pre-umbrella - # behavior: the root helper, created if absent. - defp helper_rels(_root, _project), do: [@helper_rel] + # A single app (no project, or a non-umbrella one): `mix test` requires the helper of every + # entry in the project's `:test_paths`, and Mutare reads none of the target's Mix config, so the + # paths come from the option. Each helper is created if absent. + defp helper_rels(_root, _project, test_paths), + do: Enum.map(test_paths, &Path.join(&1, @helper_name)) # Wrap the user's test helper with the selector/timeout bootstrap, the coverage # probe (split around `ExUnit.start/0`; see the constants above) and the failure @@ -475,7 +480,7 @@ defmodule Mutare.Sandbox do schema |> Schema.materialized_sources() |> Map.merge(coverage_helper_files(root, project)) - |> Map.merge(helper_files(root, project, attribution_source(options))) + |> Map.merge(helper_files(root, project, options.test_paths, attribution_source(options))) |> Map.merge(config_files(root)) |> Map.merge(RuntimeConfig.files(root, @excluded)) |> Map.merge(project_files(root, project)) @@ -583,8 +588,8 @@ defmodule Mutare.Sandbox do ) end - defp helper_files(root, project, attribution) do - Map.new(helper_rels(root, project), fn rel -> + defp helper_files(root, project, test_paths, attribution) do + Map.new(helper_rels(root, project, test_paths), fn rel -> user_helper = case File.read(Path.join(root, rel)) do {:ok, source} -> source diff --git a/test/mutare/coverage_runner_test.exs b/test/mutare/coverage_runner_test.exs index 84a7c013..3512ff9d 100644 --- a/test/mutare/coverage_runner_test.exs +++ b/test/mutare/coverage_runner_test.exs @@ -208,6 +208,55 @@ defmodule Mutare.CoverageRunnerTest do }) end + describe "custom :test_paths (end to end)" do + test "the probe records through every configured test path's helper" do + %{project: project, sandbox: sandbox} = + Project.build(:custom_test_paths, %{ + "mix.exs" => """ + defmodule CustomTestPaths.MixProject do + use Mix.Project + + def project, + do: [ + app: :custom_test_paths, + version: "0.1.0", + elixir: "~> 1.15", + test_paths: ["apps/a/test", "apps/b/test"] + ] + end + """, + "lib/calc.ex" => "defmodule Calc do\n def add(a, b), do: a + b\nend\n", + "apps/a/test/test_helper.exs" => "ExUnit.start()\n", + "apps/a/test/calc_test.exs" => """ + defmodule CalcTest do + use ExUnit.Case + test "covers add loosely", do: assert(is_integer(Calc.add(2, 3))) + end + """, + "apps/b/test/test_helper.exs" => "ExUnit.start()\n", + "apps/b/test/other_test.exs" => """ + defmodule OtherTest do + use ExUnit.Case + test "unrelated", do: assert(1 == 1) + end + """ + }) + + File.rm_rf!(Path.join(project, "test")) + + assert {:ok, run} = + Mutare.run(project, + sandbox: sandbox, + mutators: @probe, + test_paths: ["apps/a/test", "apps/b/test"] + ) + + assert [%Result{status: :survived} = result] = run.results + assert result.selection == :tests + assert tests_run(result.output) == 1 + end + end + describe "test-case selection (:tests default, end to end)" do test "the default narrows to the covering test case within a multi-test file" do %{project: project, sandbox: sandbox} = two_test_project(:tests_narrow) diff --git a/test/mutare/options/registry_test.exs b/test/mutare/options/registry_test.exs index 37fc5619..cbcc7561 100644 --- a/test/mutare/options/registry_test.exs +++ b/test/mutare/options/registry_test.exs @@ -97,6 +97,7 @@ defmodule Mutare.Options.RegistryTest do {"only_files", "(all discovered files)"}, {"only_lines", "(all lines)"}, {"test_selection", "tests"}, + {"test_paths", "[\"test\"]"}, {"workers", to_string(System.schedulers_online() |> div(2) |> min(4) |> max(1))}, {"schedulers", "#{Options.new([]).schedulers} per worker"}, {"partition_env", "(off)"}, diff --git a/test/mutare/sandbox_test.exs b/test/mutare/sandbox_test.exs index a9c5e526..32d58797 100644 --- a/test/mutare/sandbox_test.exs +++ b/test/mutare/sandbox_test.exs @@ -298,6 +298,34 @@ defmodule Mutare.SandboxTest do refute File.read!(Path.join(plain, "test/test_helper.exs")) =~ "coverage attribution" end + test "wraps the helper of every configured test path in a single app", context do + for dir <- ["apps/a/test", "apps/b/test"] do + File.mkdir_p!(Path.join(context.project, dir)) + + File.write!( + Path.join([context.project, dir, "test_helper.exs"]), + "UserHelper.#{Path.basename(Path.dirname(dir))}()\n" + ) + end + + sandbox = Path.join(context.base, "sandbox") + + Sandbox.prepare(context.project, context.schema, + sandbox: sandbox, + test_paths: ["apps/a/test", "apps/b/test"] + ) + + for {dir, call} <- [{"apps/a/test", "UserHelper.a()"}, {"apps/b/test", "UserHelper.b()"}] do + helper = File.read!(Path.join([sandbox, dir, "test_helper.exs"])) + assert helper =~ "injected by Mutare: coverage dump" + [{user, _}] = :binary.matches(helper, call) + [{dump, _}] = :binary.matches(helper, "injected by Mutare: coverage dump") + assert user < dump + end + + refute File.exists?(Path.join(sandbox, "test/test_helper.exs")) + end + test "steps the generated support app aside from a real app at its name" do %{umbrella: umbrella, sandbox: sandbox} = Umbrella.build(:support_name_taken, %{ diff --git a/usage-rules/skills/mutare/references/troubleshooting.md b/usage-rules/skills/mutare/references/troubleshooting.md index b59bc5df..18e6643a 100644 --- a/usage-rules/skills/mutare/references/troubleshooting.md +++ b/usage-rules/skills/mutare/references/troubleshooting.md @@ -124,6 +124,10 @@ suppressing the mutant. ## Slow runs +- **Every mutant runs the whole suite, and the probe reports `coverage dump unreadable + (:enoent)`.** The project sets `test_paths:` in its `mix.exs`, so `mix test` never loads + the `test/test_helper.exs` Mutare wrapped and the probe records nothing. Set the same + directories as `test_paths:` in `.mutare.exs`. - **Whole-suite fallbacks.** A mutant whose code runs in a process no test can be traced to runs the whole suite. After the coverage probe, a `↺` line says how many mutants will. `--verbose` tags each one's line `(whole suite)`, and the JSON marks