diff --git a/README.md b/README.md index 1bc220e..ecbbc1c 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,22 @@ with ProoflineClient() as client: Each response returns the original text, decoded rows, dataset version, settled horizon, and an optional next cursor. Each CSV page includes its own header and keeps cells as strings, including nested JSON cells. NDJSON pages retain JSON values. To read additional pages, repeat the call with the returned cursor and the same dataset, window, format, and grouping. A missing next cursor marks the last page. +## CI job summaries + +`outcome_markdown` turns the `ReviewOutcome` from `wait_for_review` into Markdown, and `write_github_step_summary` appends it to the GitHub Actions job summary named by `GITHUB_STEP_SUMMARY`. + +```python +import sys +from proofline import write_github_step_summary + +with ProoflineClient() as client: + outcome = client.wait_for_review(organization_id, review_id) +write_github_step_summary(outcome) +sys.exit(0 if outcome.status == "COMPLETED" else 1) +``` + +The summary shows the status, a link to the review, the finding count, each assessment gap with its reason codes, and each unresolved check by name, outcome and lifecycle. Every status other than `COMPLETED` is marked as not a pass. Check detail and unresolved reasons are left out because they can carry withheld environment values, so open the review to read them. Server text is escaped so it cannot break the tables or add links or HTML, and long lists stop at 50 rows. Outside GitHub Actions, when `GITHUB_STEP_SUMMARY` is unset, the writer does nothing and returns `False`. Pass `path` to write somewhere else. + ## Coverage uploads To upload coverage reports from CI, use a repository-restricted token with `coverage:write` and current repository write access. Name the full commit SHA that the tests ran on. Send 1 to 50 LCOV, Cobertura XML, JaCoCo XML, or Go cover reports in a JSON body of at most 30 MiB. diff --git a/proofline/__init__.py b/proofline/__init__.py index 61bd629..3f7a8ec 100644 --- a/proofline/__init__.py +++ b/proofline/__init__.py @@ -34,6 +34,7 @@ from .client import AsyncProoflineClient, ProoflineClient, ReviewFilters from .exports import ExportDataset, ExportFormat, ExportPage from .outcomes import AssessmentGap, CheckGap, ReviewOutcome +from .summary import outcome_markdown, write_github_step_summary __all__ = [ "AgentSession", @@ -74,6 +75,8 @@ "UploadedReport", "__version__", "models", + "outcome_markdown", + "write_github_step_summary", ] from .knowledge import KnowledgeResult as KnowledgeResult diff --git a/proofline/summary.py b/proofline/summary.py new file mode 100644 index 0000000..4f632f4 --- /dev/null +++ b/proofline/summary.py @@ -0,0 +1,113 @@ +"""Markdown summaries of review outcomes for CI job pages.""" + +from __future__ import annotations + +import os +import string +from pathlib import Path + +from .outcomes import ReviewOutcome + +_HEADINGS = { + "COMPLETED": "completed", + "INCOMPLETE": "incomplete", + "FAILED": "failed", + "STOPPED": "stopped", + "TIMED_OUT": "timed out", +} +_LEADS = { + "COMPLETED": "Every assessment finished with its evidence recorded.", + "INCOMPLETE": "The review ended without the evidence needed to call it complete.", + "FAILED": "The review failed.", + "STOPPED": "The review was stopped or superseded before it finished.", + "TIMED_OUT": "The wait ended before the review finished.", +} +_NOT_A_PASS = "Do not treat this result as a pass." +_PUNCTUATION = frozenset(string.punctuation) +_MAX_CELL = 200 +_MAX_ROWS = 50 + + +def _cell(value: object) -> str: + """Render server text as literal Markdown that cannot break a table or add HTML.""" + text = "".join(ch if ch.isprintable() else " " for ch in str(value)).strip() + if len(text) > _MAX_CELL: + text = text[: _MAX_CELL - 1] + "…" + escaped = "".join(f"\\{ch}" if ch in _PUNCTUATION else ch for ch in text) + return escaped or "—" + + +def _link(url: str) -> str: + safe = url.startswith(("https://", "http://")) and not any( + ch.isspace() or ch in "<>()[]`\\" for ch in url + ) + return f"[Open the review in Proofline]({url})" if safe else f"Review: {_cell(url)}" + + +def outcome_markdown(outcome: ReviewOutcome) -> str: + """Return a Markdown summary of a review outcome. + + The summary names the status, the review link, each assessment gap with its reason codes, + and each unresolved check by name, outcome and lifecycle. Free-text check detail and + unresolved reasons are left out because they can carry withheld environment values; + open the review to read them. Every status other than ``COMPLETED`` says it is not a pass. + """ + status = outcome.status + heading = _HEADINGS.get(status, _cell(status)) + lines = [f"### Proofline review {heading}", ""] + lead = _LEADS.get(status, "The review status is not recognized.") + if status != "COMPLETED": + lead = f"{lead} {_NOT_A_PASS}" + lines += [lead, ""] + if outcome.reason: + lines += [_cell(outcome.reason), ""] + review = outcome.review + if review is not None and type(getattr(review, "finding_count", None)) is int: + count = review.finding_count + lines += [f"Findings: {count}", ""] + lines += [_link(outcome.review_url), ""] + + if outcome.gaps: + lines += ["| Assessment | Status | Reasons |", "| --- | --- | --- |"] + shown = outcome.gaps[:_MAX_ROWS] + for gap in shown: + label = f"`{str(gap.assessment_id)[:8]}`" if gap.assessment_id else "Review" + reasons = ", ".join(_cell(code) for code in gap.reason_codes) or "—" + lines.append(f"| {label} | {_cell(gap.status)} | {reasons} |") + if len(outcome.gaps) > len(shown): + lines.append(f"| … | | {len(outcome.gaps) - len(shown)} more in the review |") + lines.append("") + + checks = [(gap, check) for gap in outcome.gaps for check in gap.checks] + if checks: + lines += [ + "| Assessment | Check | Outcome | Lifecycle |", + "| --- | --- | --- | --- |", + ] + for gap, check in checks[:_MAX_ROWS]: + label = f"`{str(gap.assessment_id)[:8]}`" if gap.assessment_id else "Review" + lines.append( + f"| {label} | {_cell(check.name)} | {_cell(check.outcome)} " + f"| {_cell(check.lifecycle)} |" + ) + if len(checks) > _MAX_ROWS: + lines.append(f"| … | {len(checks) - _MAX_ROWS} more in the review | | |") + lines.append("") + return "\n".join(lines) + + +def write_github_step_summary( + outcome: ReviewOutcome, path: str | os.PathLike[str] | None = None +) -> bool: + """Append the outcome summary to the GitHub Actions job summary. + + With no ``path``, the summary goes to the file named by ``GITHUB_STEP_SUMMARY``. Outside + GitHub Actions, when that variable is unset or empty, nothing is written and the function + returns ``False``, so the same script runs locally. Returns ``True`` after writing. + """ + target = path if path is not None else os.environ.get("GITHUB_STEP_SUMMARY") + if not target: + return False + with Path(target).open("a", encoding="utf-8") as file: + file.write(outcome_markdown(outcome) + "\n") + return True diff --git a/tests/test_summary.py b/tests/test_summary.py new file mode 100644 index 0000000..349dfd9 --- /dev/null +++ b/tests/test_summary.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +from pathlib import Path +from uuid import UUID + +import pytest + +from proofline import ( + AssessmentGap, + CheckGap, + ReviewOutcome, + outcome_markdown, + write_github_step_summary, +) + +URL = "https://proofline.sh/app/org/reviews/00000000-0000-4000-8000-000000000001" +ASSESSMENT = UUID("1234abcd-0000-4000-8000-000000000002") + + +def _gap(*checks: CheckGap, reasons: tuple[str, ...] = ("CHECKS_UNRESOLVED",)) -> AssessmentGap: + return AssessmentGap(ASSESSMENT, "COMPLETED", reasons, checks=checks) + + +def test_completed_outcome_links_the_review() -> None: + text = outcome_markdown(ReviewOutcome("COMPLETED", URL)) + + assert text.startswith("### Proofline review completed\n") + assert f"[Open the review in Proofline]({URL})" in text + assert "not a pass" not in text + assert "| Assessment |" not in text + + +@pytest.mark.parametrize("status", ["INCOMPLETE", "FAILED", "STOPPED", "TIMED_OUT"]) +def test_every_other_status_is_marked_as_not_a_pass(status: str) -> None: + text = outcome_markdown(ReviewOutcome(status, URL)) # type: ignore[arg-type] + + assert "Do not treat this result as a pass." in text + assert "### Proofline review completed" not in text + + +def test_gaps_and_unresolved_checks_are_listed() -> None: + check = CheckGap("unit tests", "FAILING", "GATING") + outcome = ReviewOutcome( + "INCOMPLETE", + URL, + gaps=(_gap(check), AssessmentGap(None, "UNKNOWN", ("ASSESSMENT_HISTORY_UNAVAILABLE",))), + ) + + text = outcome_markdown(outcome) + + assert "| `1234abcd` | COMPLETED | CHECKS\\_UNRESOLVED |" in text + assert "| Review | UNKNOWN | ASSESSMENT\\_HISTORY\\_UNAVAILABLE |" in text + assert "| `1234abcd` | unit tests | FAILING | GATING |" in text + + +def test_free_text_detail_and_unresolved_reason_are_left_out() -> None: + check = CheckGap( + "deploy", + "UNRESOLVED", + "GATING", + unresolved_reason="token=s3cret-reason", + detail="DATABASE_URL=postgres://user:s3cret-detail@db", + ) + + text = outcome_markdown(ReviewOutcome("INCOMPLETE", URL, gaps=(_gap(check),))) + + assert "s3cret" not in text + assert "deploy" in text + + +def test_server_text_cannot_break_the_table_or_inject_markup() -> None: + hostile = "a | b\n [link](https://evil.test) `code`" + check = CheckGap(hostile, "FAILING", "GATING") + + text = outcome_markdown(ReviewOutcome("INCOMPLETE", URL, gaps=(_gap(check),))) + row = next(line for line in text.splitlines() if "img" in line) + + assert row.count("|") - row.count("\\|") == 5 + assert "\\ None: + text = outcome_markdown(ReviewOutcome("COMPLETED", "javascript:alert(1)")) + + assert "](" not in text + assert "Review: javascript\\:alert\\(1\\)" in text + + +def test_long_gap_lists_are_capped() -> None: + checks = tuple(CheckGap(f"check {index}", "FAILING", "GATING") for index in range(60)) + + text = outcome_markdown(ReviewOutcome("INCOMPLETE", URL, gaps=(_gap(*checks),))) + + assert "check 49" in text + assert "check 50" not in text + assert "10 more in the review" in text + + +def test_writer_appends_to_the_step_summary( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + summary = tmp_path / "summary.md" + summary.write_text("earlier step\n", encoding="utf-8") + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + outcome = ReviewOutcome("TIMED_OUT", URL) + + assert write_github_step_summary(outcome) is True + assert summary.read_text(encoding="utf-8") == ( + "earlier step\n" + outcome_markdown(outcome) + "\n" + ) + + +def test_writer_is_a_no_op_outside_github_actions( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + monkeypatch.chdir(tmp_path) + + assert write_github_step_summary(ReviewOutcome("COMPLETED", URL)) is False + assert list(tmp_path.iterdir()) == [] + + +def test_writer_accepts_an_explicit_path(tmp_path: Path) -> None: + target = tmp_path / "out.md" + + assert write_github_step_summary(ReviewOutcome("FAILED", URL), target) is True + assert target.read_text(encoding="utf-8").startswith("### Proofline review failed")