From 8ab0b38ddbeb64bc599802b20a6ae42fb36224d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Moritz=20Gro=C3=9F?= Date: Sun, 27 Sep 2026 01:14:02 +0200 Subject: [PATCH] Create rule coverage functionality (#834) * v1 of rule coverage report * Add Python script to generate Markdown and HTML rule coverage reports * clarify name of logfiles * Switch rule coverage events to JSONL format * switch to jinja2 templates * Exclude Unicode mapping files from rule coverage report and improve report generation * Exclude definition files from rule coverage report and update related documentation * Add hit counts and test details to rule coverage report Enhance the rule coverage report to include hit counts, test details, and hoverable tooltips for matched rules. Improve report formatting and coverage fraction calculation. Update related tests and documentation to reflect these changes. * fix title * Simplify rule coverage event recording * Stream rule coverage events and trim unused report fields * Suppress unused variable warning when `rule-coverage` feature is disabled * Migrate `rule_coverage` implementation to a dedicated `rulecoverage` package, separating logic, templates, and CLI. * uv run ruff format * simplify dataclasses in Python * remove "loaded" as statistic for rule coverage * remove MD report and only keep HTML. simplify rest accordingly. * uv ruff format --- Cargo.lock | 26 +++ Cargo.toml | 2 + PythonScripts/README.md | 9 + PythonScripts/audit_translations/README.md | 3 + PythonScripts/audit_translations/cli.py | 13 +- .../rulecoverage/__init__.py | 1 + .../rulecoverage/rule_coverage.html | 93 +++++++++++ .../rulecoverage/rule_coverage.py | 116 +++++++++++++ .../rulecoverage/rule_coverage_report.py | 66 ++++++++ .../tests/test_rule_coverage_cli.py | 156 ++++++++++++++++++ PythonScripts/pyproject.toml | 1 + PythonScripts/uv.lock | 44 +++++ src/lib.rs | 2 + src/rule_coverage.rs | 58 +++++++ src/speech.rs | 12 ++ 15 files changed, 600 insertions(+), 2 deletions(-) create mode 100644 PythonScripts/audit_translations/rulecoverage/__init__.py create mode 100644 PythonScripts/audit_translations/rulecoverage/rule_coverage.html create mode 100644 PythonScripts/audit_translations/rulecoverage/rule_coverage.py create mode 100644 PythonScripts/audit_translations/rulecoverage/rule_coverage_report.py create mode 100644 PythonScripts/audit_translations/tests/test_rule_coverage_cli.py create mode 100644 src/rule_coverage.rs diff --git a/Cargo.lock b/Cargo.lock index 529029471..66743aed7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -651,6 +651,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + [[package]] name = "jiff" version = "0.2.20" @@ -805,6 +811,7 @@ dependencies = [ "radix_fmt", "regex", "roman-numerals-rs", + "serde_json", "strum", "sxd-document-no-unsafe", "sxd-xpath-no-unsafe", @@ -1362,6 +1369,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" @@ -2074,6 +2094,12 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7948af682ccbc3342b6e9420e8c51c1fe5d7bf7756002b4a3c6cabfe96a7e3c" +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + [[package]] name = "zopfli" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index e07193031..a51507536 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ exclude = ["src/main.rs", "docs", "BrailleDocs", "PythonScripts"] # should ha [features] +"rule-coverage" = ["dep:serde_json"] "include-zip" = [] "enable-logs" = ["android_logger"] "tts" = [ "natural-tts" ] @@ -38,6 +39,7 @@ cfg-if = "1.0.1" fastrand = { version = "2.3.0" } clap = { version = "4.5.60", features = ["derive"] } html-escape = "0.2.13" +serde_json = { version = "1.0", optional = true } elements_rs = { version = "0.2.8", default-features = false } # Runtime extracts inner language/braille zips (BZIP2). The package-rules binary also diff --git a/PythonScripts/README.md b/PythonScripts/README.md index 9eb6ed44e..d79d97dca 100644 --- a/PythonScripts/README.md +++ b/PythonScripts/README.md @@ -9,3 +9,12 @@ If you run from the repo root instead of inside `PythonScripts`, point uv at the uv sync --project PythonScripts uv run --project PythonScripts audit-translations de ``` + +To run rule YAML coverage from the repo root, use the audit tool: + +```bash +uv run --project PythonScripts audit-translations --rule-coverage +``` + +The tool opens `target/rule-coverage/index.html` in a browser when the run finishes. Expand a YAML file to see each active rule's name, tag, hit count, and match status. Hover over a rule to see which tests hit it. Coverage fractions include percentages. +The Rust test output remains at `target/rule-coverage/test.log`. diff --git a/PythonScripts/audit_translations/README.md b/PythonScripts/audit_translations/README.md index 5ebf83203..3e6d67add 100644 --- a/PythonScripts/audit_translations/README.md +++ b/PythonScripts/audit_translations/README.md @@ -67,11 +67,13 @@ The tool automatically adjusts its matching logic based on the file type: uv run audit-translations [--file ] uv run audit-translations --source uv run audit-translations --list +uv run audit-translations --rule-coverage # If running from the repo root, point uv at the project: uv run --project PythonScripts audit-translations uv run --project PythonScripts audit-translations --source uv run --project PythonScripts audit-translations --list +uv run --project PythonScripts audit-translations --rule-coverage ``` **Convenience Features:** @@ -82,6 +84,7 @@ uv run --project PythonScripts audit-translations --list * `--rules-dir`: Override the Rules/Languages directory path. * `--only`: Filter issue types (comma-separated): `missing`, `untranslated`, `extra`, `diffs`, `all`. * `--verbose`: Show detailed output including source/target snippets for rule differences. +* `--rule-coverage`: Run the full Rust test suite, generate file- and rule-level pattern coverage under `target/rule-coverage/`, and open the HTML report. File coverage includes files with active rules; empty and include-only files are not separate coverage targets. Expand a file to inspect matched and unmatched rules by name and tag, their hit counts, and hover over a rule to see the tests that matched it. Coverage fractions include percentages. This runs independently of a language audit. * **Summary Stats:** Provides a statistical summary after every run. **Examples:** diff --git a/PythonScripts/audit_translations/cli.py b/PythonScripts/audit_translations/cli.py index 6916a636e..81cac0dd0 100644 --- a/PythonScripts/audit_translations/cli.py +++ b/PythonScripts/audit_translations/cli.py @@ -10,6 +10,7 @@ from .auditor import audit_language, list_languages from .errors import AuditError from .renderer import console +from .rulecoverage.rule_coverage import run as run_rule_coverage def main() -> None: @@ -17,7 +18,7 @@ def main() -> None: sys.stdout.reconfigure(encoding="utf-8") parser = argparse.ArgumentParser( - description="Audit MathCAT translation files against a source language", + description="Audit MathCAT translations or report rule YAML test coverage", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: @@ -25,6 +26,7 @@ def main() -> None: uv run audit-translations nb --source sv uv run audit-translations de --file SharedRules/default.yaml uv run audit-translations --list + uv run audit-translations --rule-coverage """, ) @@ -32,6 +34,9 @@ def main() -> None: parser.add_argument("--source", default="en", help="Source/reference language code (default: 'en')") parser.add_argument("--file", dest="specific_file", help="Audit only a specific file (e.g., 'SharedRules/default.yaml')") parser.add_argument("--list", action="store_true", help="List available languages") + parser.add_argument( + "--rule-coverage", action="store_true", help="Run the full Rust tests and open the rule YAML coverage report" + ) parser.add_argument("--rules-dir", help="Override Rules/Languages directory path") parser.add_argument( "--only", @@ -45,7 +50,11 @@ def main() -> None: args = parser.parse_args() - if args.list: + if args.rule_coverage: + if args.language or args.list or args.specific_file or args.rules_dir or args.only or args.verbose or args.source != "en": + parser.error("--rule-coverage cannot be combined with translation audit options") + sys.exit(run_rule_coverage()) + elif args.list: list_languages(args.rules_dir) elif not args.language: parser.print_help() diff --git a/PythonScripts/audit_translations/rulecoverage/__init__.py b/PythonScripts/audit_translations/rulecoverage/__init__.py new file mode 100644 index 000000000..f9de05fe7 --- /dev/null +++ b/PythonScripts/audit_translations/rulecoverage/__init__.py @@ -0,0 +1 @@ +"""Rule coverage collection and report generation.""" diff --git a/PythonScripts/audit_translations/rulecoverage/rule_coverage.html b/PythonScripts/audit_translations/rulecoverage/rule_coverage.html new file mode 100644 index 000000000..c5235df79 --- /dev/null +++ b/PythonScripts/audit_translations/rulecoverage/rule_coverage.html @@ -0,0 +1,93 @@ + + + + + +Rule YAML coverage + + + +

Rule YAML coverage

+

Status: {{ status }}

+
{{ rule_file_count }} files with active rules{{ file_coverage }} rule files matched +{{ rule_coverage }} active rules matched
+

Paths are relative to Rules/. Files with active rules are coverage targets; +a file is matched when a rule from it completes its replacement successfully.

+{% if errors %} +

Problems

    +{% for error in errors %}
  • {{ error }}
    • {% endfor %} +
+{% endif %} +
+
+
+{% for file in files %} +
+{{ 'Matched' if file.matched else 'No match' }} +{{ file.path }} {{ file.coverage }} rules +

{{ file.coverage }} active rules matched

    +{% set file_index = loop.index0 %} +{% for rule in file.rules %} +
  • +{{ 'Matched' if rule.matched else 'No match' }} +{{ rule.name }} {{ rule.tag }} {{ rule.hits }} {{ 'hit' if rule.hits == 1 else 'hits' }} +{{ rule.tooltip }}
    • +{% endfor %} +
+
+{% endfor %} +
+ + + + diff --git a/PythonScripts/audit_translations/rulecoverage/rule_coverage.py b/PythonScripts/audit_translations/rulecoverage/rule_coverage.py new file mode 100644 index 000000000..a06178336 --- /dev/null +++ b/PythonScripts/audit_translations/rulecoverage/rule_coverage.py @@ -0,0 +1,116 @@ +"""Run the full Rust test suite and report which rule YAML files it exercises.""" + +import json +import shutil +import subprocess +import sys +import webbrowser +from collections import Counter, defaultdict +from pathlib import Path, PurePosixPath + +from .rule_coverage_report import RuleKey, coverage, render_html + +ROOT = Path(__file__).resolve().parents[3] +OUTPUT = ROOT / "target" / "rule-coverage" +EVENTS = OUTPUT / "events" + + +def read_events() -> tuple[set[RuleKey], dict[RuleKey, Counter[str]], list[str]]: + defined_rules: set[RuleKey] = set() + hits_by_test: dict[RuleKey, Counter[str]] = defaultdict(Counter) + errors: list[str] = [] + for event_file in sorted(EVENTS.glob("*.jsonl")): + for number, line in enumerate(event_file.read_text(encoding="utf-8").splitlines(), 1): + try: + event = json.loads(line) + except json.JSONDecodeError: + errors.append(f"Invalid JSON in {event_file.name}:{number}") + continue + if not isinstance(event, dict): + errors.append(f"Invalid event in {event_file.name}:{number}") + continue + kind = event.get("kind") + path = event.get("path") + if not isinstance(path, str): + errors.append(f"Invalid event in {event_file.name}:{number}") + continue + path = path.replace("\\", "/") + parts = PurePosixPath(path).parts + if ( + not parts + or PurePosixPath(path).is_absolute() + or ".." in parts + or PurePosixPath(path).suffix not in (".yaml", ".yml") + ): + errors.append(f"Invalid event in {event_file.name}:{number}") + elif kind in ("defined-rule", "matched-rule") and {"name", "tag"} <= event.keys(): + name, tag = event["name"], event["tag"] + if not isinstance(name, str) or not isinstance(tag, str) or not name or not tag: + errors.append(f"Empty rule identity in {event_file.name}:{number}") + elif kind == "defined-rule" and event.keys() == {"kind", "path", "name", "tag"}: + defined_rules.add((path, name, tag)) + elif ( + kind == "matched-rule" + and event.keys() == {"kind", "path", "name", "tag", "test"} + and isinstance(event["test"], str) + and event["test"] + ): + key = (path, name, tag) + hits_by_test[key][event["test"]] += 1 + else: + errors.append(f"Invalid event in {event_file.name}:{number}") + else: + errors.append(f"Invalid event in {event_file.name}:{number}") + return defined_rules, dict(hits_by_test), errors + + +def run() -> int: + """Generate the HTML report and open it when the run finishes.""" + OUTPUT.mkdir(parents=True, exist_ok=True) + (OUTPUT / "report.md").unlink(missing_ok=True) + if EVENTS.exists(): + shutil.rmtree(EVENTS) + EVENTS.mkdir() + + command = ["cargo", "test", "--features", "rule-coverage"] + log_path = OUTPUT / "test.log" + print(f"Running {' '.join(command)}; saving output to {log_path.relative_to(ROOT)}", flush=True) + with log_path.open("w", encoding="utf-8") as log: + try: + result = subprocess.run(command, cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, check=False) + test_status = result.returncode + except OSError as error: + log.write(f"Could not run cargo: {error}\n") + test_status = 1 + + defined_rules, hits_by_test, errors = read_events() + matched_rules = set(hits_by_test) + rule_files = {path for path, _, _ in defined_rules} + matched_files = {path for path, _, _ in matched_rules} + if test_status: + errors.insert(0, f"cargo test failed (exit status {test_status}); see test.log") + if not defined_rules: + errors.append("No active rule definitions found") + if not matched_rules: + errors.append("No matched rule events found") + if matched_rules - defined_rules: + errors.append("Matched rules lack definition events") + + status = "Incomplete" if errors else "Complete" + html_path = OUTPUT / "index.html" + html_path.write_text(render_html(defined_rules, hits_by_test, errors), encoding="utf-8") + print( + f"{status}: {coverage(len(matched_files), len(rule_files))} rule files matched; " + f"{coverage(len(matched_rules), len(defined_rules))} rules matched; report: {html_path}" + ) + try: + opened = webbrowser.open(html_path.resolve().as_uri()) + except OSError, webbrowser.Error: + opened = False + if not opened: + print(f"Browser unavailable; open {html_path} manually") + return 1 if errors else 0 + + +if __name__ == "__main__": + sys.exit(run()) diff --git a/PythonScripts/audit_translations/rulecoverage/rule_coverage_report.py b/PythonScripts/audit_translations/rulecoverage/rule_coverage_report.py new file mode 100644 index 000000000..e266a9cee --- /dev/null +++ b/PythonScripts/audit_translations/rulecoverage/rule_coverage_report.py @@ -0,0 +1,66 @@ +"""Format file and rule coverage events as an interactive HTML report.""" + +from collections import Counter, defaultdict +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +type RuleKey = tuple[str, str, str] + +TEMPLATE = Environment( + loader=FileSystemLoader(Path(__file__).parent), + autoescape=select_autoescape(("html",)), +).get_template("rule_coverage.html") + + +def coverage(count: int, total: int) -> str: + """Show the matched fraction and its percentage, including empty groups.""" + return f"{count}/{total} ({100 * count / total:.0f}%)" if total else "0/0 (0%)" + + +def render_html( + defined_rules: set[RuleKey], + hits_by_test: dict[RuleKey, Counter[str]], + errors: list[str], +) -> str: + """Show file coverage and searchable rule details in a standalone page.""" + rule_files = {path for path, _, _ in defined_rules} + matched_files = {path for path, _, _ in hits_by_test} + rules_by_path: dict[str, list[dict]] = defaultdict(list) + for path, name, tag in defined_rules: + key = (path, name, tag) + tests = hits_by_test.get(key, {}) + total_hits = sum(tests.values()) + rules_by_path[path].append( + { + "name": name, + "tag": tag, + "matched": key in hits_by_test, + "hits": total_hits, + "tooltip": "Tests:\n" + + "\n".join(f"{test} ({count} {'hit' if count == 1 else 'hits'})" for test, count in sorted(tests.items())) + if total_hits + else "No test hits", + } + ) + + files = [] + for path in sorted(rule_files): + rules = sorted(rules_by_path.get(path, []), key=lambda rule: (rule["name"], rule["tag"])) + files.append( + { + "path": path, + "matched": path in matched_files, + "coverage": coverage(sum(rule["matched"] for rule in rules), len(rules)), + "rules": rules, + } + ) + + return TEMPLATE.render( + status="Incomplete" if errors else "Complete", + rule_file_count=len(rule_files), + file_coverage=coverage(len(matched_files), len(rule_files)), + rule_coverage=coverage(len(hits_by_test), len(defined_rules)), + errors=errors, + files=files, + ) diff --git a/PythonScripts/audit_translations/tests/test_rule_coverage_cli.py b/PythonScripts/audit_translations/tests/test_rule_coverage_cli.py new file mode 100644 index 000000000..33709d1e6 --- /dev/null +++ b/PythonScripts/audit_translations/tests/test_rule_coverage_cli.py @@ -0,0 +1,156 @@ +"""Checks the rule coverage command without rerunning the Rust test suite.""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from .. import cli +from ..rulecoverage import rule_coverage + + +def test_coverage_command_generates_html_and_opens_browser(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The CLI writes only the HTML report, removes an old Markdown report, and opens the page.""" + output = tmp_path / "target" / "rule-coverage" + output.mkdir(parents=True) + (output / "report.md").write_text("old report", encoding="utf-8") + monkeypatch.setattr(rule_coverage, "ROOT", tmp_path) + monkeypatch.setattr(rule_coverage, "OUTPUT", output) + monkeypatch.setattr(rule_coverage, "EVENTS", output / "events") + + def fake_cargo(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + assert command == ["cargo", "test", "--features", "rule-coverage"] + assert kwargs["cwd"] == tmp_path + events = [ + {"kind": "defined-rule", "path": "Languages/en/SimpleSpeak_Rules.yaml", "name": "simple", "tag": "mi"}, + {"kind": "defined-rule", "path": "Languages/en/SimpleSpeak_Rules.yaml", "name": "default", "tag": "mi"}, + {"kind": "defined-rule", "path": "Languages/en/Z_Rules.yaml", "name": "unused", "tag": "mn"}, + { + "kind": "matched-rule", + "path": "Languages/en/SimpleSpeak_Rules.yaml", + "name": "simple", + "tag": "mi", + "test": "test_alpha", + }, + { + "kind": "matched-rule", + "path": "Languages/en/SimpleSpeak_Rules.yaml", + "name": "simple", + "tag": "mi", + "test": "test_alpha", + }, + { + "kind": "matched-rule", + "path": "Languages/en/SimpleSpeak_Rules.yaml", + "name": "simple", + "tag": "mi", + "test": "test_beta", + }, + ] + (output / "events" / "pid-123.jsonl").write_text( + "\n".join(json.dumps(event) for event in events) + "\n", + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0) + + opened: list[str] = [] + monkeypatch.setattr(rule_coverage.subprocess, "run", fake_cargo) + monkeypatch.setattr(rule_coverage.webbrowser, "open", lambda url: opened.append(url) or True) + monkeypatch.setattr(sys, "argv", ["audit-translations", "--rule-coverage"]) + + with pytest.raises(SystemExit) as result: + cli.main() + + assert result.value.code == 0 + assert opened == [(output / "index.html").as_uri()] + assert not (output / "report.md").exists() + html = (output / "index.html").read_text(encoding="utf-8") + assert "Status: Complete" in html + assert "1/2 (50%) rule files matched" in html + assert "Languages/en/Z_Rules.yaml" in html + assert 'data-rule-search="simple mi" data-status="matched"' in html + assert 'data-rule-search="default mi" data-status="unmatched"' in html + assert "1/3 (33%) active rules matched" in html + assert "1/2 (50%) rules" in html + assert 'role="tooltip">Tests:\ntest_alpha (2 hits)\ntest_beta (1 hit)' in html + assert ".rules li:hover .rule-tooltip, .rules li:focus .rule-tooltip { display: block; }" in html + assert 'aria-describedby="rule-tooltip-0-1"' in html + assert "3 hits" in html + assert "0 hits" in html + assert "2 files with active rules" in html + assert (output / "test.log").is_file() + + +def test_failed_coverage_run_opens_incomplete_report(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A failed test run remains unsuccessful but still produces and opens a useful report.""" + output = tmp_path / "target" / "rule-coverage" + monkeypatch.setattr(rule_coverage, "ROOT", tmp_path) + monkeypatch.setattr(rule_coverage, "OUTPUT", output) + monkeypatch.setattr(rule_coverage, "EVENTS", output / "events") + monkeypatch.setattr( + rule_coverage.subprocess, + "run", + lambda command, **kwargs: subprocess.CompletedProcess(command, 1), + ) + opened: list[str] = [] + monkeypatch.setattr(rule_coverage.webbrowser, "open", lambda url: opened.append(url) or True) + + assert rule_coverage.run() == 1 + assert opened == [(output / "index.html").as_uri()] + html = (output / "index.html").read_text(encoding="utf-8") + assert "Status: Incomplete" in html + assert "No active rule definitions found" in html + assert not (output / "report.md").exists() + + +def test_jsonl_rule_identity_preserves_separators_and_unicode(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """JSONL rule names survive tabs, newlines, and Unicode without splitting events.""" + events = tmp_path / "events" + events.mkdir() + monkeypatch.setattr(rule_coverage, "EVENTS", events) + name = "fraction\tname\nüber" + path = "Languages/en/SimpleSpeak_Rules.yaml" + (events / "pid-123.jsonl").write_text( + json.dumps({"kind": "defined-rule", "path": path, "name": name, "tag": "mfrac"}) + "\n", + encoding="utf-8", + ) + + defined, hits_by_test, errors = rule_coverage.read_events() + + assert hits_by_test == {} + assert defined == {(path, name, "mfrac")} + assert errors == [] + + +def test_invalid_and_unknown_rule_events_make_report_incomplete(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Malformed events and hits without definitions must keep the report incomplete.""" + output = tmp_path / "target" / "rule-coverage" + monkeypatch.setattr(rule_coverage, "ROOT", tmp_path) + monkeypatch.setattr(rule_coverage, "OUTPUT", output) + monkeypatch.setattr(rule_coverage, "EVENTS", output / "events") + path = "Languages/en/SimpleSpeak_Rules.yaml" + + def fake_cargo(command: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + events = [ + {"kind": "loaded", "path": path}, + {"kind": "defined-rule", "path": path, "name": "known", "tag": "mi"}, + {"kind": "matched-rule", "path": path, "name": "unknown", "tag": "mi", "test": "test_alpha"}, + ] + (output / "events" / "pid-123.jsonl").write_text( + "\n".join(json.dumps(event) for event in events) + "\n{bad json\n", + encoding="utf-8", + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(rule_coverage.subprocess, "run", fake_cargo) + monkeypatch.setattr(rule_coverage.webbrowser, "open", lambda url: True) + + assert rule_coverage.run() == 1 + html = (output / "index.html").read_text(encoding="utf-8") + assert "Status: Incomplete" in html + assert "Invalid event in pid-123.jsonl:1" in html + assert "Invalid JSON in pid-123.jsonl:4" in html + assert "Matched rules lack definition events" in html + assert not (output / "report.md").exists() diff --git a/PythonScripts/pyproject.toml b/PythonScripts/pyproject.toml index 7b9adf68d..e1f579a31 100644 --- a/PythonScripts/pyproject.toml +++ b/PythonScripts/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "deepl", "googletrans", "html-table-extractor", + "jinja2>=3.1.6", "jsonpath-ng>=1.8.0", "pyyaml", "rich", diff --git a/PythonScripts/uv.lock b/PythonScripts/uv.lock index 81e0404c4..d2323646c 100644 --- a/PythonScripts/uv.lock +++ b/PythonScripts/uv.lock @@ -200,6 +200,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + [[package]] name = "jsonpath-ng" version = "1.8.0" @@ -221,6 +233,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + [[package]] name = "mdurl" version = "0.1.2" @@ -282,6 +324,7 @@ dependencies = [ { name = "deepl" }, { name = "googletrans" }, { name = "html-table-extractor" }, + { name = "jinja2" }, { name = "jsonpath-ng" }, { name = "pyyaml" }, { name = "rich" }, @@ -300,6 +343,7 @@ requires-dist = [ { name = "deepl" }, { name = "googletrans" }, { name = "html-table-extractor" }, + { name = "jinja2", specifier = ">=3.1.6" }, { name = "jsonpath-ng", specifier = ">=1.8.0" }, { name = "pyyaml" }, { name = "rich" }, diff --git a/src/lib.rs b/src/lib.rs index ac577a47f..97cf10d08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,8 @@ mod prefs; mod tts; mod xpath_functions; mod definitions; +#[cfg(feature = "rule-coverage")] +mod rule_coverage; pub mod pretty_print; mod chemistry; diff --git a/src/rule_coverage.rs b/src/rule_coverage.rs new file mode 100644 index 000000000..f0277d5ca --- /dev/null +++ b/src/rule_coverage.rs @@ -0,0 +1,58 @@ +//! Per-process YAML rule coverage events, enabled only by `rule-coverage`. + +use std::collections::HashSet; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; + +struct Recorder { + output: File, + recorded: HashSet<(PathBuf, String, String)>, +} + +static RECORDER: OnceLock> = OnceLock::new(); + +fn recorder() -> &'static Mutex { + RECORDER.get_or_init(|| { + let event_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target/rule-coverage/events"); + fs::create_dir_all(&event_dir).expect("cannot create rule coverage event directory"); + let event_file = event_dir.join(format!("pid-{}.jsonl", std::process::id())); + let output = OpenOptions::new().create(true).append(true).open(event_file) + .expect("cannot open rule coverage event file"); + Mutex::new(Recorder { output, recorded: HashSet::new() }) + }) +} + +fn rule_relative_path(path: &Path) -> Option { + let rules_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("Rules"); + let relative = path.strip_prefix(&rules_dir) + .or_else(|_| path.strip_prefix("Rules")) + .ok()?; + if relative.as_os_str().is_empty() || !matches!(relative.extension().and_then(|s| s.to_str()), Some("yaml" | "yml")) { + return None; + } + Some(relative.to_path_buf()) +} + +pub(crate) fn defined_rule(path: &Path, name: &str, tag: &str) { + let Some(relative) = rule_relative_path(path) else { return }; + let mut recorder = recorder().lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if !recorder.recorded.insert((relative.clone(), name.to_string(), tag.to_string())) { + return; + } + let path = relative.to_string_lossy(); + let event = serde_json::json!({"kind": "defined-rule", "path": path, "name": name, "tag": tag}); + writeln!(recorder.output, "{event}").expect("cannot write rule coverage event"); +} + +pub(crate) fn matched_rule(path: &Path, name: &str, tag: &str) { + let Some(relative) = rule_relative_path(path) else { return }; + let mut recorder = recorder().lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let event = serde_json::json!({ + "kind": "matched-rule", "path": relative.to_string_lossy(), "name": name, "tag": tag, + "test": std::thread::current().name().unwrap_or("(unnamed thread)"), + }); + writeln!(recorder.output, "{event}").expect("cannot write rule coverage event"); +} diff --git a/src/speech.rs b/src/speech.rs index 4b6691bec..b667cb612 100644 --- a/src/speech.rs +++ b/src/speech.rs @@ -2291,6 +2291,14 @@ impl SpeechRules { if self.rules.is_empty() || !self.rule_files.is_file_up_to_date(&rule_file, should_ignore_file_time) { self.rules.clear(); let files_read = self.read_patterns(&rule_file)?; + #[cfg(feature = "rule-coverage")] + for patterns in self.rules.values() { + for pattern in patterns { + crate::rule_coverage::defined_rule( + Path::new(&pattern.file_name), &pattern.pattern_name, &pattern.tag_name + ); + } + } self.rule_files.set_files_and_times(files_read); } @@ -2511,6 +2519,10 @@ impl<'c, 's:'c, 'r, 'm:'c> SpeechRulesWithContext<'c, 's,'m> { } return match result { Ok(s) => { + #[cfg(feature = "rule-coverage")] + crate::rule_coverage::matched_rule( + Path::new(&pattern.file_name), &pattern.pattern_name, &pattern.tag_name + ); // for all except braille and navigation, nav_node_id will be an empty string and will not match if self.nav_node_id.is_empty() { Ok( Some(s) )