From 207bf9d8911d16c0231c693326f31ccd413e663e Mon Sep 17 00:00:00 2001 From: iray-tno Date: Sun, 14 Jun 2026 10:29:23 +0900 Subject: [PATCH] feat: #11 add relative suite scoring --- src/commands/test_command.py | 120 ++++++++++++++++++++++++++++++++++- tests/test_test_command.py | 87 +++++++++++++++++++++++++ 2 files changed, 206 insertions(+), 1 deletion(-) diff --git a/src/commands/test_command.py b/src/commands/test_command.py index a37ef42..7a5ae55 100644 --- a/src/commands/test_command.py +++ b/src/commands/test_command.py @@ -1,6 +1,7 @@ """Test command for competitive programming.""" import click import concurrent.futures +import json import tempfile import sys import os @@ -56,8 +57,16 @@ def single_cmd(file, input_file, expected_file, preprocess, include_paths, rust) @click.option('--preprocess', is_flag=True, help='Preprocess the file before compiling') @click.option('-I', '--include-path', 'include_paths', multiple=True, type=click.Path(exists=True, file_okay=False, resolve_path=True), help='Include paths for preprocessing') @click.option('--rust', is_flag=True, help='Force Rust mode (auto-detected from .rs extension)') -def suite_cmd(file, cases_dir, pattern, workers, timeout, preprocess, include_paths, rust): +@click.option('--best-known', 'best_known_file', type=click.Path(dir_okay=False, resolve_path=True), help='JSON file with best-known numeric outputs by case') +@click.option('--update-best-known', is_flag=True, help='Update --best-known with improved numeric outputs from this run') +@click.option('--score-mode', type=click.Choice(['min', 'max']), default='min', show_default=True, help='Whether lower or higher numeric outputs are better') +@click.option('--relative-scale', type=float, default=1.0, show_default=True, help='Per-case relative score scale') +def suite_cmd(file, cases_dir, pattern, workers, timeout, preprocess, include_paths, rust, best_known_file, update_best_known, score_mode, relative_scale): """Compile once and run an exact-match test suite over *.in/*.out cases.""" + if update_best_known and not best_known_file: + click.echo("❌ --update-best-known requires --best-known", err=True) + sys.exit(1) + config = load_config() timeout = timeout if timeout is not None else config.get("test", {}).get("timeout", 5) @@ -102,6 +111,14 @@ def suite_cmd(file, cases_dir, pattern, workers, timeout, preprocess, include_pa click.echo(f" {result['error']}") click.echo(f"\nSummary: {passed}/{len(results)} passed") + if best_known_file: + best_known = _load_best_known(best_known_file) + _report_relative_scores(results, best_known, score_mode, relative_scale) + if update_best_known: + updated = _update_best_known(results, best_known, score_mode) + _write_best_known(best_known_file, best_known) + click.echo(f"\nUpdated best-known: {best_known_file} ({updated} case(s) improved)") + if passed != len(results): sys.exit(1) finally: @@ -205,6 +222,7 @@ def _run_suite_case(executable, input_file, expected_file, timeout): "passed": False, "runtime": timeout, "error": f"timed out after {timeout}s", + "output": "", } if run_result.returncode != 0: @@ -213,6 +231,7 @@ def _run_suite_case(executable, input_file, expected_file, timeout): "passed": False, "runtime": runtime, "error": f"runtime error: {run_result.stderr.strip()}", + "output": run_result.stdout, } passed = run_result.stdout.strip() == expected_output.strip() @@ -222,9 +241,108 @@ def _run_suite_case(executable, input_file, expected_file, timeout): "passed": passed, "runtime": runtime, "error": error, + "output": run_result.stdout, } +def _load_best_known(best_known_file): + if not os.path.exists(best_known_file): + return {} + with open(best_known_file, 'r') as f: + data = json.load(f) + if not isinstance(data, dict): + raise click.ClickException("--best-known must contain a JSON object") + return data + + +def _write_best_known(best_known_file, best_known): + parent_dir = os.path.dirname(best_known_file) + if parent_dir: + os.makedirs(parent_dir, exist_ok=True) + with open(best_known_file, 'w') as f: + json.dump(best_known, f, indent=2, sort_keys=True) + f.write("\n") + + +def _report_relative_scores(results, best_known, score_mode, relative_scale): + click.echo("\n--- Relative Score ---") + total = 0.0 + scored = 0 + for result in results: + case = result["case"] + try: + your_score = _parse_numeric_output(result["output"]) + except ValueError as exc: + click.echo(f"{case}: N/A ({exc})") + continue + + best_score = best_known.get(case) + if best_score is None: + click.echo(f"{case}: N/A (no best-known value)") + continue + + try: + relative_score = _relative_score(your_score, float(best_score), score_mode, relative_scale) + except ValueError as exc: + click.echo(f"{case}: N/A ({exc})") + continue + + total += relative_score + scored += 1 + click.echo( + f"{case}: your={_format_score(your_score)} " + f"best={_format_score(float(best_score))} " + f"relative={relative_score:.6f}" + ) + + click.echo(f"Total relative score: {total:.6f} / {(scored * relative_scale):.6f}") + + +def _update_best_known(results, best_known, score_mode): + updated = 0 + for result in results: + try: + your_score = _parse_numeric_output(result["output"]) + except ValueError: + continue + + current_best = best_known.get(result["case"]) + if current_best is None or _is_better(your_score, float(current_best), score_mode): + best_known[result["case"]] = your_score + updated += 1 + return updated + + +def _parse_numeric_output(output): + tokens = output.strip().split() + if not tokens: + raise ValueError("empty output") + try: + return float(tokens[0]) + except ValueError as exc: + raise ValueError("first output token is not numeric") from exc + + +def _relative_score(your_score, best_score, score_mode, relative_scale): + if your_score <= 0 or best_score <= 0: + raise ValueError("relative score requires positive numeric values") + if score_mode == "min": + return relative_scale * best_score / your_score + return relative_scale * your_score / best_score + + +def _is_better(candidate, current_best, score_mode): + if score_mode == "min": + return candidate < current_best + return candidate > current_best + + +def _format_score(score): + if score.is_integer(): + return str(int(score)) + return f"{score:.6f}" + + def _test_cpp(file, input_file, expected_file, preprocess, include_paths): """Test C++ code.""" # Load config for defaults diff --git a/tests/test_test_command.py b/tests/test_test_command.py index 09af231..55dfcbe 100644 --- a/tests/test_test_command.py +++ b/tests/test_test_command.py @@ -254,6 +254,93 @@ def test_cpp_suite_reports_failure(self, tmp_path): assert "output differs from expected" in result.output assert "Summary: 0/1 passed" in result.output + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") + def test_cpp_suite_reports_relative_score_from_best_known(self, tmp_path): + """Test suite relative scoring against best-known values.""" + test_file = tmp_path / "echo_first.cpp" + test_file.write_text(""" +#include + +int main() { + int value; + std::cin >> value; + std::cout << value << std::endl; + return 0; +} +""") + + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "001.in").write_text("10\n") + (cases_dir / "001.out").write_text("10\n") + (cases_dir / "002.in").write_text("20\n") + (cases_dir / "002.out").write_text("20\n") + best_known = tmp_path / "best.json" + best_known.write_text('{"001.in": 5, "002.in": 10}\n') + + runner = CliRunner() + result = runner.invoke( + cli, + [ + 'test', + 'suite', + str(test_file), + str(cases_dir), + '--best-known', + str(best_known), + '--relative-scale', + '100', + ], + ) + + assert result.exit_code == 0 + assert "--- Relative Score ---" in result.output + assert "001.in: your=10 best=5 relative=50.000000" in result.output + assert "002.in: your=20 best=10 relative=50.000000" in result.output + assert "Total relative score: 100.000000 / 200.000000" in result.output + + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") + def test_cpp_suite_updates_best_known(self, tmp_path): + """Test best-known JSON updates when this run improves numeric values.""" + test_file = tmp_path / "echo_first.cpp" + test_file.write_text(""" +#include + +int main() { + int value; + std::cin >> value; + std::cout << value << std::endl; + return 0; +} +""") + + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "001.in").write_text("5\n") + (cases_dir / "001.out").write_text("5\n") + (cases_dir / "002.in").write_text("7\n") + (cases_dir / "002.out").write_text("7\n") + best_known = tmp_path / "best.json" + best_known.write_text('{"001.in": 10}\n') + + runner = CliRunner() + result = runner.invoke( + cli, + [ + 'test', + 'suite', + str(test_file), + str(cases_dir), + '--best-known', + str(best_known), + '--update-best-known', + ], + ) + + assert result.exit_code == 0 + assert f"Updated best-known: {best_known} (2 case(s) improved)" in result.output + assert best_known.read_text() == '{\n "001.in": 5.0,\n "002.in": 7.0\n}\n' + class TestTestCommandErrorHandling: """Test error handling for the test command."""