diff --git a/src/commands/test_command.py b/src/commands/test_command.py index 22d042d..ed64acf 100644 --- a/src/commands/test_command.py +++ b/src/commands/test_command.py @@ -22,6 +22,8 @@ def test(args): suite_args = list(args[1:]) if suite_args and suite_args[0] == "compare": suite_compare_cmd.main(args=suite_args[1:], prog_name="test suite compare", standalone_mode=False) + elif suite_args and suite_args[0] == "noise-floor": + suite_noise_floor_cmd.main(args=suite_args[1:], prog_name="test suite noise-floor", standalone_mode=False) else: suite_cmd.main(args=suite_args, prog_name="test suite", standalone_mode=False) else: @@ -201,6 +203,54 @@ def suite_compare_cmd(file_a, file_b, cases_dir, pattern, workers, runs, timeout os.remove(executable) +@click.command(name="noise-floor") +@click.argument('file', type=click.Path(exists=True, resolve_path=True)) +@click.argument('cases_dir', type=click.Path(exists=True, file_okay=False, resolve_path=True)) +@click.option('--pattern', default='*.in', show_default=True, help='Input filename glob inside cases_dir') +@click.option('-j', '--workers', default=1, show_default=True, type=click.IntRange(min=1), help='Parallel paired worker count') +@click.option('--runs', default=3, show_default=True, type=click.IntRange(min=1), help='Self-comparison pairs per case') +@click.option('--timeout', type=float, help='Per-run timeout in seconds (defaults to prepkit_config.yaml test.timeout or 5)') +@click.option('--preprocess', is_flag=True, help='Preprocess the file before compiling') +@click.option('-I', '--include-path', 'include_paths', multiple=True, type=click.Path(exists=True, file_okay=False, resolve_path=True), help='Include paths for preprocessing') +@click.option('--rust', is_flag=True, help='Force Rust mode (auto-detected from .rs extension)') +def suite_noise_floor_cmd(file, cases_dir, pattern, workers, runs, timeout, preprocess, include_paths, rust): + """Measure same-solver run-to-run score variance.""" + config = load_config() + timeout = timeout if timeout is not None else config.get("test", {}).get("timeout", 5) + is_rust = _detect_suite_language(file, rust) + + cases = _discover_suite_cases(cases_dir, pattern) + if not cases: + click.echo(f"❌ No cases found in {cases_dir} matching {pattern} with .out files", err=True) + sys.exit(1) + + executable = None + source_file = None + try: + source_file = _prepare_source_for_compile(file, preprocess, include_paths, is_rust, config) + executable = _compile_for_suite(file, source_file, is_rust, config) + + click.echo(f"Calibrating {len(cases)} case(s) x {runs} self-pair(s) with {workers} worker(s)...") + pair_results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = [ + executor.submit(_run_compare_pair, executable, executable, input_file, expected_file, timeout) + for input_file, expected_file in cases + for _ in range(runs) + ] + for future in concurrent.futures.as_completed(futures): + pair_results.append(future.result()) + + _report_noise_floor_results(pair_results) + if any(not result["valid"] for result in pair_results): + sys.exit(1) + finally: + if preprocess and source_file and os.path.exists(source_file): + os.remove(source_file) + if executable and os.path.exists(executable): + os.remove(executable) + + def _detect_suite_language(file, force_rust): file_ext = os.path.splitext(file)[1].lower() is_rust = force_rust or file_ext == '.rs' @@ -398,6 +448,37 @@ def _report_compare_results(pair_results, score_mode): click.echo("Diff stdev: 0.000000") +def _report_noise_floor_results(pair_results): + pair_results.sort(key=lambda result: result["case"]) + valid_results = [result for result in pair_results if result["valid"]] + + click.echo("\n--- Noise Floor ---") + for result in pair_results: + if not result["valid"]: + click.echo(f"{result['case']}: N/A ({result['error']})") + continue + click.echo( + f"{result['case']}: " + f"run1={_format_score(result['score_a'])} " + f"run2={_format_score(result['score_b'])} " + f"abs-diff={abs(result['diff']):.6f}" + ) + + if not valid_results: + click.echo("\nSummary: 0 valid self-pair(s)") + return + + diffs = [result["diff"] for result in valid_results] + abs_diffs = [abs(diff) for diff in diffs] + click.echo(f"\nSummary: {len(valid_results)} valid self-pair(s)") + click.echo(f"Mean absolute diff: {statistics.mean(abs_diffs):.6f}") + click.echo(f"Max absolute diff: {max(abs_diffs):.6f}") + if len(diffs) > 1: + click.echo(f"Signed diff stdev: {statistics.stdev(diffs):.6f}") + else: + click.echo("Signed diff stdev: 0.000000") + + def _compare_winner(score_a, score_b, score_mode): if score_a == score_b: return "tie" diff --git a/tests/test_test_command.py b/tests/test_test_command.py index 9f2fa7c..d4ef63a 100644 --- a/tests/test_test_command.py +++ b/tests/test_test_command.py @@ -447,6 +447,48 @@ def test_cpp_suite_compare_reports_paired_wins(self, tmp_path): assert "Summary: A wins 4/4, B wins 0/4, ties 0/4" in result.output assert "Mean diff(A-B): -5.000000" in result.output + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") + def test_cpp_suite_noise_floor_reports_self_pair_variance(self, tmp_path): + """Test same-solver noise-floor calibration for numeric outputs.""" + test_file = tmp_path / "echo_first.cpp" + test_file.write_text(""" +#include + +int main() { + int value; + std::cin >> value; + std::cout << value << std::endl; + return 0; +} +""") + + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "001.in").write_text("10\n") + (cases_dir / "001.out").write_text("10\n") + + runner = CliRunner() + result = runner.invoke( + cli, + [ + 'test', + 'suite', + 'noise-floor', + str(test_file), + str(cases_dir), + '--runs', + '2', + ], + ) + + assert result.exit_code == 0 + assert "Calibrating 1 case(s) x 2 self-pair(s) with 1 worker(s)..." in result.output + assert "--- Noise Floor ---" in result.output + assert "001.in: run1=10 run2=10 abs-diff=0.000000" in result.output + assert "Summary: 2 valid self-pair(s)" in result.output + assert "Mean absolute diff: 0.000000" in result.output + assert "Max absolute diff: 0.000000" in result.output + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") def test_cpp_suite_updates_best_known(self, tmp_path): """Test best-known JSON updates when this run improves numeric values."""