diff --git a/CHANGELOG.md b/CHANGELOG.md index d1bc541..90f9544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ breaks with no more of its own users than it held alone: it competes with the others for something they share (a database pool, CPU, a cache). Saved runs carry a `combined` field. `--no-combined` skips it. +- **Journeys everywhere.** The `--html` report shows each journey, the combined run and + `--expect`; the MCP tools' run summaries list each journey's verdict; and `prescale compare` + diffs each journey by name (terminal, `--json`, `--markdown`). A drop past the threshold in any + journey counts as a regression for `--fail-on-regression`, even when it isn't the weakest journey. - **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split across journeys by weight: each must hold its share alone, and all together must hold N. diff --git a/README.md b/README.md index 2bed172..19d7d27 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,8 @@ Interference checkout broke with 15 of its own users, but held 20 alone. It competes with quickbuy for something they share (a database pool, CPU, a cache). ``` +`prescale show`, `--html` reports and the MCP tools show journeys too, and `prescale compare` diffs each journey: a drop past the threshold in any journey is a regression (so `--fail-on-regression` catches it), even when it isn't the weakest one; it's marked confident when the confidence bands don't overlap. A journey that held everything it was tested to, below the old level, is inconclusive, not a drop. A URL run and a journey run measure different things, so comparing them is reported as not comparable. + `--no-combined` skips this pass. `--expect 500` says whether you'd hold 500 users at peak: in a journey run, each journey must hold its share of them (by weight) on its own, and all journeys together must hold 500; a journey that wasn't load-tested makes the answer unknown. It reports, it doesn't change the exit code (use `--fail-under` to gate CI). - `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. diff --git a/cli/src/prescale_cli/commands/compare.py b/cli/src/prescale_cli/commands/compare.py index a6cb951..9b90454 100644 --- a/cli/src/prescale_cli/commands/compare.py +++ b/cli/src/prescale_cli/commands/compare.py @@ -6,11 +6,18 @@ import click from rich.console import Console +from rich.markup import escape from rich.panel import Panel from rich.table import Table from prescale_cli.bars import text_bar -from prescale_cli.compare import compare_results, resolve, to_markdown +from prescale_cli.compare import ( + compare_results, + journey_change, + journey_users, + resolve, + to_markdown, +) from prescale_cli.result import AmbiguousResultError, ResultNotFoundError console = Console() @@ -97,3 +104,18 @@ def _render(cmp: dict) -> None: note = "" if (cmp["confident"] or not cmp["regressed"]) else \ " (confidence bands overlap — could be noise)" console.print(Panel(cmp["summary"] + note, border_style=color)) + if cmp.get("journeys"): + jt = Table(show_header=True, header_style="bold magenta", title="Per journey") + for col, justify in (("Journey", "left"), ("Old", "right"), ("New", "right"), + ("Change", "right"), ("Breaks at", "left")): + jt.add_column(col, justify=justify) + for j in cmp["journeys"]: + if j["status"] != "ok": + jt.add_row(escape(j["name"]), "", "", f"[dim]{escape(j['status'])}[/dim]", "") + continue + tone = "red" if j["regressed"] else ("green" if j.get("improved") else "") + change = journey_change(j) + jt.add_row(escape(j["name"]), journey_users(j, "old"), journey_users(j, "new"), + f"[{tone}]{change}[/{tone}]" if tone else change, + escape(j.get("breaks_at") or "—")) + console.print(jt) diff --git a/cli/src/prescale_cli/compare.py b/cli/src/prescale_cli/compare.py index dea8b2e..6f9ae91 100644 --- a/cli/src/prescale_cli/compare.py +++ b/cli/src/prescale_cli/compare.py @@ -40,6 +40,44 @@ def compare_results(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHO if o["mode"] != n["mode"]: # URL capacity and journey capacity measure different things: never a regression. return _not_comparable(o, n) + d = _diff(o, n, threshold) + cmp = { + "old": o, + "new": n, + "survives_delta": n["survives_users"] - o["survives_users"], + "survives_pct": round(d["pct"], 3), + "peak_rps_pct": (round((n["peak_rps"] - o["peak_rps"]) / o["peak_rps"], 3) + if o["peak_rps"] else 0.0), + "regressed": d["regressed"], + "improved": d["improved"], + "inconclusive": d["inconclusive"], + "confident": d["confident"], + "summary": (_summary(o, n, d["pct"], d["regressed"], d["improved"], d["inconclusive"]) + + _journey_note(o, n)), + } + if o["mode"] == "journeys": + cmp["journeys"] = compare_journeys(old, new, threshold=threshold) + worse = [j for j in cmp["journeys"] if j.get("regressed")] + if worse: + # A drop in any one journey is a regression, even if it isn't the weakest. + # Confidence only comes from drops: a confident overall gain says nothing + # about whether a journey's drop is noise. + cmp["regressed"] = True + cmp["confident"] = ((d["confident"] and d["regressed"]) + or any(j["confident"] for j in worse)) + if not d["regressed"]: + cmp["improved"] = False + cmp["summary"] = "Capacity regressed: " + "; ".join( + f"{j['name']} {j['old']} → {j['new']} users ({j['pct']:+.0%})" + for j in worse) + "." + if d["improved"]: + cmp["summary"] += (f" (Overall: {o['survives_users']} → " + f"{n['survives_users']} users, {d['pct']:+.0%}.)") + return cmp + + +def _diff(o: dict, n: dict, threshold: float) -> dict: + """The capacity change between two sets of facts (a run, or one journey).""" o_surv, n_surv = o["survives_users"], n["survives_users"] pct = (n_surv - o_surv) / o_surv if o_surv else 0.0 inconclusive = (n["onset_users"] is None and n["max_tested"] is not None @@ -51,19 +89,64 @@ def compare_results(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHO confident = n["survives_high"] < o["survives_low"] elif improved and o["survives_high"] is not None and n["survives_low"] is not None: confident = n["survives_low"] > o["survives_high"] - return { - "old": o, - "new": n, - "survives_delta": n_surv - o_surv, - "survives_pct": round(pct, 3), - "peak_rps_pct": (round((n["peak_rps"] - o["peak_rps"]) / o["peak_rps"], 3) - if o["peak_rps"] else 0.0), - "regressed": regressed, - "improved": improved, - "inconclusive": inconclusive, - "confident": confident, - "summary": _summary(o, n, pct, regressed, improved, inconclusive) + _journey_note(o, n), - } + return {"pct": pct, "regressed": regressed, "improved": improved, + "inconclusive": inconclusive, "confident": confident} + + +def _journey_facts(j: dict) -> dict: + conf = j.get("confidence") or {} + return {"survives_users": j["survives_users"], "survives_low": conf.get("survives_low"), + "survives_high": conf.get("survives_high"), "onset_users": j["onset_users"], + "max_tested": j.get("max_tested")} + + +def compare_journeys(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHOLD) -> list: + """Per-journey capacity change, matched by name. A journey that is new, gone, + or not load-tested on either side is listed with its status, never as a drop.""" + before = {j["name"]: j for j in old.get("journeys") or []} + after = {j["name"]: j for j in new.get("journeys") or []} + rows = [] + for name in [*before, *(n for n in after if n not in before)]: + o, n = before.get(name), after.get(name) + if o is None or n is None: + rows.append({"name": name, "status": "new" if o is None else "removed", + "regressed": False, "confident": False}) + continue + if o["status"] != "ok" or n["status"] != "ok": + rows.append({"name": name, "status": f"{o['status']} → {n['status']}", + "regressed": False, "confident": False}) + continue + d = _diff(_journey_facts(o), _journey_facts(n), threshold) + step = (n.get("culprit_step") or {}).get("name") + rows.append({"name": name, "status": "ok", "old": o["survives_users"], + "new": n["survives_users"], "pct": round(d["pct"], 3), + "old_held_all": o["onset_users"] is None, + "new_held_all": n["onset_users"] is None, + "regressed": d["regressed"], "improved": d["improved"], + "inconclusive": d["inconclusive"], "confident": d["confident"], + "breaks_at": step}) + return rows + + +def journey_users(row: dict, side: str) -> str: + """A journey's capacity in a compare row: "~20", or "200+" if it held every level.""" + return f"{row[side]}+" if row.get(f"{side}_held_all") else f"~{row[side]}" + + +def journey_change(row: dict, *, markers: bool = False) -> str: + """The change cell: the percentage, or "inconclusive" when the new run held + everything it tested but was never pushed as far as the old one.""" + if row.get("inconclusive"): + return "inconclusive" + mark = "" + if markers: + mark = " 🔴" if row["regressed"] else (" 🟢" if row.get("improved") else "") + return f"{row['pct']:+.0%}{mark}" + + +def _cell(text: str) -> str: + """Text safe inside a markdown table cell (journey and step names come from YAML).""" + return str(text).replace("|", "\\|").replace("\n", " ") def _not_comparable(o: dict, n: dict) -> dict: @@ -117,7 +200,7 @@ def to_markdown(cmp: dict) -> str: emoji = "🟢" else: emoji = "⚪" - return "\n".join([ + lines = [ f"### {emoji} PreScale — {cmp['summary']}", "", "| | survives | peak req/s | commit |", @@ -132,4 +215,15 @@ def to_markdown(cmp: dict) -> str: max(o["survives_users"], n["survives_users"], 1) * 1.1, ), "```", - ]) + ] + if cmp.get("journeys"): + lines += ["", "| journey | old | new | change | breaks at |", "|---|---:|---:|---:|---|"] + for j in cmp["journeys"]: + name = _cell(j["name"]) + if j["status"] != "ok": + lines.append(f"| {name} | | | {j['status']} | |") + continue + step = _cell(j.get("breaks_at") or "—") + lines.append(f"| {name} | {journey_users(j, 'old')} | {journey_users(j, 'new')} | " + f"{journey_change(j, markers=True)} | {step} |") + return "\n".join(lines) diff --git a/cli/src/prescale_cli/mcp_tools.py b/cli/src/prescale_cli/mcp_tools.py index 1278e87..da3e646 100644 --- a/cli/src/prescale_cli/mcp_tools.py +++ b/cli/src/prescale_cli/mcp_tools.py @@ -77,10 +77,40 @@ def summarize_result(result: dict) -> dict: "peak_rps": v["peak_rps"], "saturated": v["saturated"], "warning": result.get("warning"), + **_journey_summary(result), "hint": "Call get_run(id) for the full per-level / per-route breakdown.", } +def _journey_summary(result: dict) -> dict: + """Journey runs: one line per journey, plus interference and --expect, kept short.""" + journeys = result.get("journeys") + if not journeys: + return {} + out: dict = {"culprit_journey": result["verdict"].get("culprit_journey"), "journeys": []} + for j in journeys: + if j["status"] != "ok": + out["journeys"].append({"name": j["name"], "status": j["status"], + "reason": j.get("reason")}) + continue + step = j.get("culprit_step") + out["journeys"].append({ + "name": j["name"], "status": "ok", + "survives_users": j["survives_users"], "held_all": j["onset_users"] is None, + "breaks_at_step": (f"{step['index']}. {step['name']} ({step['route']})" + if step else None), + "bottleneck": j.get("bottleneck"), + }) + combined = result.get("combined") + if combined: + out["together"] = {"survives_users": combined["survives_users"], + "held_all": combined["onset_users"] is None, + "interference": combined.get("interference")} + if result.get("expect"): + out["expect"] = {k: result["expect"].get(k) for k in ("users", "meets", "untested")} + return out + + def _verdict_line(v: dict, conf: dict) -> str: if v["onset_users"] is None: return f"Held up through {v['max_tested']} concurrent users (the most tested)." diff --git a/cli/src/prescale_cli/report.py b/cli/src/prescale_cli/report.py index 5de319f..07720b6 100644 --- a/cli/src/prescale_cli/report.py +++ b/cli/src/prescale_cli/report.py @@ -80,6 +80,10 @@ .cause .lbl{font-size:11px;font-weight:500;text-transform:uppercase;letter-spacing:.5px; color:var(--amber);margin-bottom:6px;} .cause p{font-size:13.5px;color:var(--muted);} +.cause ul{margin:4px 0 0 18px;font-size:13.5px;color:var(--muted);} +.note{font-size:12.5px;color:var(--muted);margin:-4px 0 8px;} +td.t,th.t{text-align:left;} +td.n{white-space:nowrap;} .cause p+p{margin-top:8px;} .cause b{color:var(--ink);} footer{margin-top:44px;padding-top:20px;border-top:1px solid var(--hair);display:flex; @@ -262,6 +266,81 @@ def _cause(result: dict) -> str: return f'
Likely cause
{"".join(paras)}
' +def _journeys_html(result: dict) -> str: + """Journey runs: each journey's verdict, the combined pass, and --expect.""" + journeys = result.get("journeys") + if not journeys: + return "" + first = result["verdict"].get("culprit_journey") + rows = [] + for j in journeys: + if j["status"] != "ok": + label = "stale" if j["status"] == "stale" else "couldn't run" + rows.append(f"{_esc(j['name'])}{label}" + f"—" + f"{_esc(j.get('reason') or '')}") + continue + if j["onset_users"] is None: + survives = f"{j['max_tested']}+ (held)" + else: + survives = f"~{j['survives_users']}{_band(j)}" + step = j.get("culprit_step") + where = (f"{step['index']}. {_esc(step['name'])} {_esc(step['route'])}" + if step else "—") + cls = ' class="onset"' if j["name"] == first and j["onset_users"] is not None else "" + rows.append(f"{_esc(j['name'])}{survives}" + f"{where}" + f"{_esc(j.get('bottleneck') or '')}") + parts = [ + '

Journeys

' + '

Each journey ramped on its own.

' + '
' + "" + + "".join(rows) + "
JourneySurvivesBreaks at stepWhy
" + ] + c = result.get("combined") + if c: + mix = ", ".join(f"{_esc(n)} {pct:g}%" for n, pct in c["mix"].items()) + paras = [f"

All journeys together ({mix}): " + + (f"held through {c['max_tested']} users in total.

" + if c["onset_users"] is None else + f"survives ~{c['survives_users']} users in total" + + (f"; breaks first at {_esc(c['culprit_journey'])}." + if c.get("culprit_journey") else ".") + "

")] + inter = c.get("interference") + if inter: + paras.append( + f"

Interference: {_esc(inter['journey'])} broke with " + f"{inter['users_at_break']} of its own users, but held {inter['held_alone']} " + f"alone. It competes with {_esc(', '.join(inter['with']))} for something " + "they share (a database pool, CPU, a cache).

") + parts.append('
Together
' + + "".join(paras) + "
") + parts.append(_expect_html(result.get("expect"))) + parts.append("
") + return "".join(parts) + + +def _expect_html(exp: dict | None) -> str: + if not exp: + return "" + answer = {True: "yes", False: "no", None: "unknown"}[exp["meets"]] + items = [] + for j in exp.get("journeys") or []: + mark = {True: "holds", False: "doesn't hold", None: "unknown"}[j["meets"]] + items.append(f"
  • {_esc(j['name'])}: ~{j['users']:g} users, holds " + f"~{j['survives_users']} alone ({mark})
    • ") + combined = exp.get("combined") + if combined: + items.append(f"
  • all together: holds ~{combined['survives_users']}
    • ") + for name in exp.get("untested") or []: + items.append(f"
  • {_esc(name)}: not load-tested (stale or unreachable)
    • ") + if not exp.get("journeys"): + items.append(f"
  • survives ~{exp['survives_users']}
    • ") + return (f'
    Expected ~{exp["users"]} users: {answer}' + f"
      {''.join(items)}
    ") + + def render_html(result: dict) -> str: target = result["target"] config = result["config"] @@ -307,6 +386,7 @@ def render_html(result: dict) -> str: {_cause(result)} {_route_table(result)} + {_journeys_html(result)}
    Generated by prescale {result["tool_version"]} {meta}
    diff --git a/cli/tests/test_journeys_report.py b/cli/tests/test_journeys_report.py new file mode 100644 index 0000000..6bfe27c --- /dev/null +++ b/cli/tests/test_journeys_report.py @@ -0,0 +1,148 @@ +"""Journey results everywhere: compare, the HTML report, and the MCP summary.""" + +import copy + +from prescale_cli.compare import compare_results, to_markdown +from prescale_cli.mcp_tools import summarize_result +from prescale_cli.report import render_html + + +def _journey(name, survives, onset, low=None, high=None, step=None, status="ok"): + if status != "ok": + return {"name": name, "type": "browse", "weight": 5, "status": status, + "reason": "step 1 (home): returned 404, expected 200"} + return {"name": name, "type": "checkout", "weight": 5, "status": "ok", "reason": None, + "survives_users": survives, "max_tested": 200, "onset_users": onset, + "onset_reason": "errors" if onset else None, + "confidence": {"survives_low": low if low is not None else survives, + "survives_high": high if high is not None else survives, + "stable": True}, + "culprit_step": ({"index": 3, "name": step, "route": "POST /api/orders"} + if step else None), + "bottleneck": "5xx under load" if onset else None} + + +def _run(journeys, survives=20, culprit="checkout"): + return { + "id": "r1", "created_at": "2026-11-02T10:00:00Z", "tool_version": "1.1.3", + "target": {"url": "http://shop.test", "host": "shop.test", "routes": ["GET /"]}, + "config": {"method": "GET", "max_users": 200, "stage_seconds": 2, + "latency_wall_s": 2.0, "error_threshold": 0.02, "max_rps": None, + "mode": "journeys"}, + "verdict": {"survives_users": survives, "max_tested": 200, "onset_users": survives + 10, + "onset_reason": "errors", "culprit_route": "POST /api/orders", + "culprit_journey": culprit, "bottleneck": "5xx under load", + "saturated": False, "saturation_users": None, "peak_rps": 300.0, + "marginal": False, + "confidence": {"survives_low": survives, "survives_high": survives, + "stable": True}}, + "stages": [{"users": 1, "rps": 3.0, "p50_ms": 20, "p95_ms": 30, "p99_ms": 40, + "error_rate": 0.0, "errors": 0, "total": 6, "routes": {}}], + "environment": {"git_commit": "abc1234"}, + "journeys": journeys, + } + + +def test_compare_flags_a_drop_in_any_journey(): + old = _run([_journey("browse", 200, None), _journey("checkout", 40, 50, step="place order")], + survives=40) + new = _run([_journey("browse", 200, None), _journey("checkout", 20, 30, step="place order")], + survives=20) + cmp = compare_results(old, new) + rows = {j["name"]: j for j in cmp["journeys"]} + assert rows["checkout"]["regressed"] and rows["checkout"]["confident"] + assert rows["checkout"]["pct"] == -0.5 and rows["checkout"]["breaks_at"] == "place order" + assert not rows["browse"]["regressed"] + assert cmp["regressed"] + assert "| checkout | ~40 | ~20 | -50% 🔴 | place order |" in to_markdown(cmp) + + +def test_a_drop_in_a_journey_that_isnt_the_weakest_is_still_a_regression(): + # search is the weakest both times; checkout drops but stays above it + old = _run([_journey("search", 10, 20), _journey("checkout", 100, 120)], survives=10, + culprit="search") + new = _run([_journey("search", 10, 20), _journey("checkout", 50, 60)], survives=10, + culprit="search") + cmp = compare_results(old, new) + assert cmp["survives_delta"] == 0 + assert cmp["regressed"] and cmp["confident"] + assert cmp["summary"] == "Capacity regressed: checkout 100 → 50 users (-50%)." + + +def test_new_removed_and_stale_journeys_are_listed_not_counted(): + old = _run([_journey("browse", 200, None), _journey("gone", 30, 40)]) + new = _run([_journey("browse", 200, None, status="stale"), _journey("fresh", 30, 40)]) + rows = {j["name"]: j["status"] for j in compare_results(old, new)["journeys"]} + assert rows == {"browse": "ok → stale", "gone": "removed", "fresh": "new"} + assert not any(j["regressed"] for j in compare_results(old, new)["journeys"]) + + +def test_html_report_shows_journeys_combined_and_expect(): + run = _run([_journey("checkout", 20, 30, low=20, high=30, step="place order"), + _journey("old", 0, None, status="stale")]) + run["combined"] = {"mix": {"checkout": 50.0, "quickbuy": 50.0}, "survives_users": 20, + "max_tested": 200, "onset_users": 30, "culprit_journey": "checkout", + "interference": {"journey": "checkout", "users_at_break": 15, + "held_alone": 20, "with": ["quickbuy"]}} + run["expect"] = {"users": 60, "meets": False, "survives_users": 20, + "journeys": [{"name": "checkout", "users": 30.0, "survives_users": 20, + "meets": False}], + "combined": {"survives_users": 20, "meets": False}, "untested": ["old"]} + html = render_html(run) + assert "

    Journeys

    " in html + assert "~20 (20–30)" in html and "3. place order" in html + assert "checkout broke with 15 of its own users, but held 20 alone" in html + assert "Expected ~60 users: no" in html and "old: not load-tested" in html + + +def test_html_escapes_journey_names_and_reasons(): + j = _journey("x", 20, 30, step="") + html = render_html(_run([j])) + assert "" not in html + + +def test_url_runs_render_and_compare_as_before(): + run = _run([]) + del run["journeys"] + run["config"].pop("mode") + assert "

    Journeys

    " not in render_html(run) + assert "journeys" not in compare_results(run, copy.deepcopy(run)) + assert "journeys" not in summarize_result(run) + + +def test_mcp_summary_lists_journeys_compactly(): + run = _run([_journey("checkout", 20, 30, step="place order"), + _journey("old", 0, None, status="stale")]) + s = summarize_result(run) + assert s["culprit_journey"] == "checkout" + assert s["journeys"][0]["breaks_at_step"] == "3. place order (POST /api/orders)" + assert s["journeys"][1] == {"name": "old", "status": "stale", + "reason": "step 1 (home): returned 404, expected 200"} + + +def test_a_confident_overall_gain_doesnt_make_a_noisy_journey_drop_confident(): + old = _run([_journey("browse", 100, 120, low=80, high=120)], survives=20) + new = _run([_journey("browse", 75, 90, low=70, high=110)], survives=40) + old["verdict"]["confidence"] = {"survives_low": 20, "survives_high": 20} + new["verdict"]["confidence"] = {"survives_low": 40, "survives_high": 40} + cmp = compare_results(old, new) + assert cmp["regressed"] and not cmp["confident"] and not cmp["improved"] + assert cmp["summary"] == ("Capacity regressed: browse 100 → 75 users (-25%). " + "(Overall: 20 → 40 users, +100%.)") + assert to_markdown(cmp).startswith("### 🟠") + + +def test_inconclusive_and_held_journeys_read_as_such(): + old = _run([_journey("browse", 200, None)]) + new = _run([_journey("browse", 50, None)]) + for run, tested in ((old, 200), (new, 50)): + run["journeys"][0]["max_tested"] = tested + md = to_markdown(compare_results(old, new)) + assert "| browse | 200+ | 50+ | inconclusive | — |" in md + + +def test_pipes_in_names_dont_break_the_markdown_table(): + old = _run([_journey("a|b", 40, 50, step="pay | ship")]) + new = _run([_journey("a|b", 20, 30, step="pay | ship")]) + md = to_markdown(compare_results(old, new)) + assert "| a\\|b | ~40 | ~20 | -50% 🔴 | pay \\| ship |" in md