From 0344572d7d8b8f259543308c7a9f9da8ad730040 Mon Sep 17 00:00:00 2001 From: Ansh Bajpai Date: Sat, 25 Jul 2026 12:19:54 +0530 Subject: [PATCH] Add a Markdown reporter for PR comments and CI summaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --markdown report.md writes a GitHub-flavored report: summary, top-functions table, and — when --compare is given — a comparison table with regression/improvement icons, new/removed lists and a regression banner. Drops straight into a PR comment or $GITHUB_STEP_SUMMARY. The reporter is a pure consumer of the ComparisonData from #90; it reimplements no comparison logic. To make the comparison available at export time, the compare step is split into _load_comparison / _export_results / _report_comparison, with no change to existing flags, output or exit codes. Co-Authored-By: Claude Fable 5 --- oracletrace/cli.py | 63 ++++++++++++----- oracletrace/reporters/__init__.py | 3 +- oracletrace/reporters/markdown.py | 91 ++++++++++++++++++++++++ tests/test_cli.py | 41 +++++++++++ tests/test_markdown_reporter.py | 111 ++++++++++++++++++++++++++++++ 5 files changed, 292 insertions(+), 17 deletions(-) create mode 100644 oracletrace/reporters/markdown.py create mode 100644 tests/test_markdown_reporter.py diff --git a/oracletrace/cli.py b/oracletrace/cli.py index 5c59ef6..d895784 100644 --- a/oracletrace/cli.py +++ b/oracletrace/cli.py @@ -13,7 +13,7 @@ from typing import List, Dict, Any, Optional, Tuple from json import JSONDecodeError -from .reporters import generate_html_report +from .reporters import generate_html_report, generate_markdown_report from .runner import execute_command, UnsupportedCommandError from .compare import compare_traces, ComparisonData from .display import print_comparison @@ -74,6 +74,7 @@ def _compile_ignore_patterns( _BASE_PARSER.add_argument("--json", help="Export trace result to JSON file") _BASE_PARSER.add_argument("--csv", help="Export trace result to CSV file") _BASE_PARSER.add_argument("--html", help="Export trace result to interactive HTML file") +_BASE_PARSER.add_argument("--markdown", help="Export a Markdown report (PR-comment / CI-summary ready)") _BASE_PARSER.add_argument("--compare", help="Compare against previous trace JSON") _BASE_PARSER.add_argument( "--ignore", @@ -112,6 +113,7 @@ def _compile_ignore_patterns( _BASELINE_SAVE_PARSER: ArgumentParser = ArgumentParser(add_help=False) _BASELINE_SAVE_PARSER.add_argument("--csv", help="Export trace result to CSV file") _BASELINE_SAVE_PARSER.add_argument("--html", help="Export trace result to interactive HTML file") +_BASELINE_SAVE_PARSER.add_argument("--markdown", help="Export a Markdown report (PR-comment / CI-summary ready)") _BASELINE_SAVE_PARSER.add_argument( "--ignore", metavar="REGEX", @@ -137,7 +139,11 @@ def _set_default(obj: Any) -> Any: raise TypeError -def _export_results(data: TracerData, args: Namespace) -> None: +def _export_results( + data: TracerData, + args: Namespace, + comparison: Optional[ComparisonData] = None, +) -> None: if args.json: with open(args.json, "w", encoding="utf-8") as f: json.dump(asdict(data), f, indent=4, default=_set_default) @@ -160,41 +166,55 @@ def _export_results(data: TracerData, args: Namespace) -> None: generate_html_report(data, args.html) print(f"HTML report generated: {os.path.abspath(args.html)}") + if getattr(args, "markdown", None): + generate_markdown_report(data, args.markdown, comparison=comparison, top=args.top) + print(f"Markdown report generated: {os.path.abspath(args.markdown)}") -def _compare_and_fail( + +def _load_comparison( data: TracerData, args: Namespace -) -> Optional[int]: +) -> Tuple[Optional[int], Optional[ComparisonData]]: + """Load the --compare baseline and run the comparison. Returns + (error_code, None) on any failure, (None, None) when no baseline was + requested, or (None, comparison) on success. Pure of side effects + beyond printing errors — the display and the exit-code gate live in the + caller so exports can reuse the same comparison.""" if not args.compare: - return None + return None, None if not os.path.exists(args.compare): print(f"Compare file not found: {args.compare}", file=sys.stderr) - return 1 + return 1, None try: with open(args.compare, "r", encoding="utf-8") as f: loaded_data = json.load(f) except JSONDecodeError as e: print(f"Compare file contains invalid JSON: {e}", file=sys.stderr) - return 1 + return 1, None except OSError as e: print(f"Compare file read error: {e}", file=sys.stderr) - return 1 + return 1, None try: old_data: TracerData = TracerData.from_dict(loaded_data) except (KeyError, TypeError, ValueError) as e: print(f"Compare file has invalid schema: {e}", file=sys.stderr) - return 1 + return 1, None comparison_result: ComparisonData = compare_traces( old_data, data, threshold=args.threshold, ) - print_comparison(comparison_result, only_regressions=args.only_regressions) + return None, comparison_result - if args.fail_on_regression and comparison_result.has_regression: + +def _report_comparison(comparison: ComparisonData, args: Namespace) -> Optional[int]: + """Print the comparison and apply the regression gate.""" + print_comparison(comparison, only_regressions=args.only_regressions) + + if args.fail_on_regression and comparison.has_regression: print( f"Build failed: performance regression above {args.threshold:.2f}% detected.", file=sys.stderr, @@ -253,9 +273,14 @@ def _init_tracer(args: Namespace) -> Tuple[Optional[int], Optional[Tracer]]: def _finish_trace(tracer: Tracer, args: Namespace) -> Optional[int]: data = tracer.get_trace_data() - _export_results(data, args) + err, comparison = _load_comparison(data, args) + if err is not None: + return err + _export_results(data, args, comparison) tracer.show_results(args.top) - return _compare_and_fail(data, args) + if comparison is not None: + return _report_comparison(comparison, args) + return None def _trace_script( @@ -358,9 +383,15 @@ def _run_target() -> int: assert data is not None if tracer is None: - _export_results(data, args) - exit_code = _compare_and_fail(data, args) - return exit_code if exit_code is not None else script_exit + err, comparison = _load_comparison(data, args) + if err is not None: + return err + _export_results(data, args, comparison) + if comparison is not None: + gate = _report_comparison(comparison, args) + if gate is not None: + return gate + return script_exit exit_code = _finish_trace(tracer, args) return exit_code if exit_code is not None else script_exit diff --git a/oracletrace/reporters/__init__.py b/oracletrace/reporters/__init__.py index afa0a28..1d7b8ed 100644 --- a/oracletrace/reporters/__init__.py +++ b/oracletrace/reporters/__init__.py @@ -1,3 +1,4 @@ from .html import generate_html_report +from .markdown import generate_markdown_report, render_markdown -__all__ = ["generate_html_report"] +__all__ = ["generate_html_report", "generate_markdown_report", "render_markdown"] diff --git a/oracletrace/reporters/markdown.py b/oracletrace/reporters/markdown.py new file mode 100644 index 0000000..1b07d18 --- /dev/null +++ b/oracletrace/reporters/markdown.py @@ -0,0 +1,91 @@ +"""GitHub-flavored Markdown reporter — drop straight into a PR comment, +CI log, or $GITHUB_STEP_SUMMARY. +""" + +from datetime import datetime +from typing import List, Optional + +from ..tracer import TracerData +from ..compare import ComparisonData, REGRESSION, IMPROVEMENT + + +def _md_escape(text: str) -> str: + # keep names from breaking out of table cells / inline code + return text.replace("|", "\\|").replace("`", "\\`") + + +def render_markdown( + data: TracerData, + comparison: Optional[ComparisonData] = None, + top: Optional[int] = None, +) -> str: + meta = data.metadata + parts: List[str] = [] + + parts.append("## ⚡ OracleTrace Performance Report\n") + parts.append( + f"**Total time:** {meta.total_execution_time:.4f}s  |  " + f"**Functions:** {meta.total_functions}  |  " + f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n" + ) + + if comparison is not None: + parts.append("### Comparison vs. baseline\n") + changed = [ + d for d in comparison.deltas + if d.status in (REGRESSION, IMPROVEMENT) + and d.old_time is not None and d.new_time is not None + ] + if changed: + parts.append("| Function | Baseline | Current | Change |") + parts.append("|---|---:|---:|---:|") + for d in sorted(changed, key=lambda d: d.percent or 0, reverse=True): + icon = "🔴" if d.status == REGRESSION else "🟢" + parts.append( + f"| `{_md_escape(d.name)}` | {d.old_time:.4f}s | {d.new_time:.4f}s " + f"| {icon} {d.percent:+.1f}% |" + ) + parts.append("") + else: + parts.append("No functions changed beyond the threshold.\n") + + if comparison.new_functions: + names = ", ".join(f"`{_md_escape(n)}`" for n in comparison.new_functions) + parts.append(f"**New:** {names}\n") + if comparison.removed_functions: + names = ", ".join(f"`{_md_escape(n)}`" for n in comparison.removed_functions) + parts.append(f"**Removed:** {names}\n") + + if comparison.regressions: + parts.append( + f"> 🚨 **{len(comparison.regressions)} regression(s)** above " + f"{comparison.threshold:.1f}% threshold\n" + ) + else: + parts.append( + f"> ✅ No regressions above {comparison.threshold:.1f}% threshold\n" + ) + + parts.append("### Top functions by total time\n") + parts.append("| Function | Total Time (s) | Calls | Avg (ms) |") + parts.append("|---|---:|---:|---:|") + for fn in sorted(data.functions, key=lambda f: f.total_time, reverse=True)[:top]: + parts.append( + f"| `{_md_escape(fn.name)}` | {fn.total_time:.4f} " + f"| {fn.call_count} | {fn.avg_time * 1000:.3f} |" + ) + parts.append("") + + parts.append("*Generated by [OracleTrace](https://github.com/KaykCaputo/oracletrace)*") + return "\n".join(parts) + "\n" + + +def generate_markdown_report( + data: TracerData, + output_path: str, + comparison: Optional[ComparisonData] = None, + top: Optional[int] = None, +) -> None: + content = render_markdown(data, comparison=comparison, top=top) + with open(output_path, "w", encoding="utf-8") as f: + f.write(content) diff --git a/tests/test_cli.py b/tests/test_cli.py index d2cbd8d..3f777b5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -185,6 +185,47 @@ def fake_run_path(path, run_name): assert sys.path[0] == str(target.parent.resolve()) +def test_main_exports_markdown(monkeypatch, tmp_path, trace_data): + target = tmp_path / "target.py" + target.write_text("print('hello')\n", encoding="utf-8") + md_output = tmp_path / "report.md" + + monkeypatch.setattr(cli, "Tracer", lambda root, ignore_patterns: FakeTracer(root, ignore_patterns, trace_data)) + monkeypatch.setattr(cli.runpy, "run_path", lambda *args, **kwargs: None) + + exit_code = _run_cli( + monkeypatch, + ["oracletrace", str(target), "--markdown", str(md_output)], + ) + + assert exit_code == 0 + content = md_output.read_text(encoding="utf-8") + assert content.startswith("## ⚡ OracleTrace Performance Report") + assert "foo" in content + + +def test_main_markdown_includes_comparison(monkeypatch, tmp_path, trace_data, baseline_trace_data): + target = tmp_path / "target.py" + target.write_text("print('hello')\n", encoding="utf-8") + compare_file = tmp_path / "baseline.json" + compare_file.write_text(json.dumps(asdict(baseline_trace_data), default=set_default), encoding="utf-8") + md_output = tmp_path / "report.md" + + monkeypatch.setattr(cli, "Tracer", lambda root, ignore_patterns: FakeTracer(root, ignore_patterns, trace_data)) + monkeypatch.setattr(cli.runpy, "run_path", lambda *args, **kwargs: None) + + exit_code = _run_cli( + monkeypatch, + ["oracletrace", str(target), "--compare", str(compare_file), "--markdown", str(md_output)], + ) + + assert exit_code == 0 + content = md_output.read_text(encoding="utf-8") + # foo went 1.5s -> 2.0s, a regression, so the comparison section renders + assert "### Comparison vs. baseline" in content + assert "🔴" in content + + def test_main_passes_top_value_to_show_results(monkeypatch, tmp_path, trace_data): target = tmp_path / "target.py" target.write_text("print('hello')\n", encoding="utf-8") diff --git a/tests/test_markdown_reporter.py b/tests/test_markdown_reporter.py new file mode 100644 index 0000000..9e10427 --- /dev/null +++ b/tests/test_markdown_reporter.py @@ -0,0 +1,111 @@ +import pytest +from oracletrace.tracer import TracerData, TracerMetadata, FunctionData +from oracletrace.compare import compare_traces +from oracletrace.reporters import render_markdown, generate_markdown_report + + +@pytest.fixture +def trace_data() -> TracerData: + return TracerData( + metadata=TracerMetadata( + root_path="/proj", + total_functions=2, + total_execution_time=2.5, + ), + functions=[ + FunctionData(name="app.py:main", total_time=2.0, call_count=1, avg_time=2.0, callees=set()), + FunctionData(name="app.py:helper", total_time=0.5, call_count=4, avg_time=0.125, callees=set()), + ], + ) + + +@pytest.fixture +def baseline_data() -> TracerData: + return TracerData( + metadata=TracerMetadata(root_path="/proj", total_functions=2, total_execution_time=2.0), + functions=[ + FunctionData(name="app.py:main", total_time=1.0, call_count=1, avg_time=1.0, callees=set()), + FunctionData(name="app.py:helper", total_time=0.5, call_count=4, avg_time=0.125, callees=set()), + ], + ) + + +class TestRenderMarkdown: + def test_header_and_summary(self, trace_data): + md = render_markdown(trace_data) + assert md.startswith("## ⚡ OracleTrace Performance Report") + assert "**Total time:** 2.5000s" in md + assert "**Functions:** 2" in md + + def test_top_functions_table_sorted_desc(self, trace_data): + md = render_markdown(trace_data) + assert "| Function | Total Time (s) | Calls | Avg (ms) |" in md + main_pos = md.index("app.py:main") + helper_pos = md.index("app.py:helper") + assert main_pos < helper_pos # hottest first + + def test_top_limit(self, trace_data): + md = render_markdown(trace_data, top=1) + assert "app.py:main" in md + # helper appears only in the top table, which is truncated to 1 + assert md.count("app.py:helper") == 0 + + def test_no_comparison_section_without_baseline(self, trace_data): + md = render_markdown(trace_data) + assert "Comparison vs. baseline" not in md + + def test_pipe_in_name_is_escaped(self): + data = TracerData( + metadata=TracerMetadata(root_path="/p", total_functions=1, total_execution_time=1.0), + functions=[FunctionData(name="a.py:f|g", total_time=1.0, call_count=1, avg_time=1.0, callees=set())], + ) + md = render_markdown(data) + assert "f\\|g" in md + assert "| a.py:f|g |" not in md # raw pipe would break the table + + +class TestComparisonSection: + def test_regression_rendered_with_icon(self, baseline_data, trace_data): + comparison = compare_traces(baseline_data, trace_data, threshold=5.0) + md = render_markdown(trace_data, comparison=comparison) + assert "### Comparison vs. baseline" in md + assert "🔴" in md + assert "app.py:main" in md + assert "+100.0%" in md + assert "🚨 **1 regression(s)** above 5.0% threshold" in md + + def test_no_regression_shows_green_banner(self, baseline_data): + comparison = compare_traces(baseline_data, baseline_data, threshold=5.0) + md = render_markdown(baseline_data, comparison=comparison) + assert "✅ No regressions above 5.0% threshold" in md + + def test_new_and_removed_functions_listed(self): + old = TracerData( + metadata=TracerMetadata(root_path="/p", total_functions=1, total_execution_time=1.0), + functions=[FunctionData(name="a.py:gone", total_time=1.0, call_count=1, avg_time=1.0, callees=set())], + ) + new = TracerData( + metadata=TracerMetadata(root_path="/p", total_functions=1, total_execution_time=1.0), + functions=[FunctionData(name="a.py:fresh", total_time=1.0, call_count=1, avg_time=1.0, callees=set())], + ) + comparison = compare_traces(old, new, threshold=5.0) + md = render_markdown(new, comparison=comparison) + assert "**New:** `a.py:fresh`" in md + assert "**Removed:** `a.py:gone`" in md + + +class TestGenerateMarkdownReport: + def test_writes_file(self, trace_data, tmp_path): + out = tmp_path / "report.md" + generate_markdown_report(trace_data, str(out)) + content = out.read_text(encoding="utf-8") + assert content.startswith("## ⚡ OracleTrace Performance Report") + assert "app.py:main" in content + + def test_self_contained_no_external_urls_except_footer(self, trace_data, tmp_path): + out = tmp_path / "report.md" + generate_markdown_report(trace_data, str(out)) + content = out.read_text(encoding="utf-8") + # the only link is the OracleTrace footer + assert content.count("https://") == 1 + assert "github.com/KaykCaputo/oracletrace" in content