diff --git a/CHANGELOG.md b/CHANGELOG.md index 6204b14..d1bc541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,13 @@ break first, with per-step timings, and saved runs carry a `journeys` field (plus `verdict.culprit_journey`). Stale journeys are skipped with the reason. `--journey` picks journeys; `--urls` tests URLs as before. +- **Journeys together, and interference.** After ramping each journey alone, `prescale run` + ramps them all together, the users split between journeys by `weight`, and flags a journey that + breaks with no more of its own users than it held alone: it competes with the others for + something they share (a database pool, CPU, a cache). Saved runs carry a `combined` field. + `--no-combined` skips it. +- **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split + across journeys by weight: each must hold its share alone, and all together must hold N. ### Fixed - **`prescale mcp` works on a fresh install again.** `pip install 'prescale[mcp]'` picked up the new diff --git a/README.md b/README.md index 16371a8..2bed172 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,15 @@ When `prescale/journeys/` has journeys, **`prescale run` replays them at load**: A stale journey isn't load-tested (a failure with no load says nothing about capacity); it's listed with the reason. `--journey NAME` picks journeys, and `--urls` (or `--path`, `--from-sitemap`) tests URLs instead. +Then **all journeys run together**, the users split between journeys by `weight` (each user stays on its journey, so a weight is that journey's share of the users in the app at once) (defaults by type: browse 60, search 25, account 8, checkout 5, write 5, auth 2; edit them in the files). That catches **interference**: journeys competing for something they share. If a journey breaks with fewer of its own users than it held alone, the report says so: + +``` +Interference checkout broke with 15 of its own users, but held 20 alone. + It competes with quickbuy for something they share (a database pool, CPU, a cache). +``` + +`--no-combined` skips this pass. `--expect 500` says whether you'd hold 500 users at peak: in a journey run, each journey must hold its share of them (by weight) on its own, and all journeys together must hold 500; a journey that wasn't load-tested makes the answer unknown. It reports, it doesn't change the exit code (use `--fail-under` to gate CI). + - `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. - **Secrets never go in the file.** A literal password, token, API key or `Authorization` value is refused; use `{{env.NAME}}`. - Paths start with `/` and are relative to the URL you give (`http://host/app` + `/api/cart` is `http://host/app/api/cart`). A journey only calls that app: a request or redirect to another host fails the step. diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index 078b832..0dbe69c 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -21,7 +21,13 @@ from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica from prescale_cli.commands.upload import header_option, upload_after_run, upload_option -from prescale_cli.journeys.load import JourneyRun, first_to_break, journeys_block, ramp +from prescale_cli.journeys.load import ( + JourneyRun, + combined_block, + first_to_break, + journeys_block, + ramp, +) from prescale_cli.journeys.model import Journey, load_dir from prescale_cli.journeys.replay import check from prescale_cli.live import LiveRamp, ramp_progress @@ -89,6 +95,11 @@ help="Frame the run as a launch scenario (see `prescale profiles`).") @click.option("--journey", "only", multiple=True, help="Replay only this journey (repeatable). See `prescale journeys`.") +@click.option("--no-combined", "no_combined", is_flag=True, + help="Skip ramping all journeys together (by weight) after each one alone.") +@click.option("--expect", "expect", default=None, type=click.IntRange(min=1), metavar="N", + help="Users you expect at peak; the report says whether you'd hold them. " + "It doesn't change the exit code (use --fail-under for a CI gate).") @click.option("--urls", "urls_only", is_flag=True, help="Test URLs, not journeys, even when prescale/journeys/ has some.") @click.option("--allow-writes", "allow_writes", multiple=True, metavar="HOST", @@ -106,7 +117,8 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, repeat: int, think_time: float, ignore_robots: bool, yes: bool, as_json: bool, html_path: str | None, store: str | None, no_save: bool, fail_under: int | None, - profile_name: str | None, only: tuple[str, ...], urls_only: bool, + profile_name: str | None, only: tuple[str, ...], no_combined: bool, + expect: int | None, urls_only: bool, allow_writes: tuple[str, ...], directory: Path, app: str | None, app_timeout: float, host_permission: bool, headers: dict[str, str], upload: bool) -> None: @@ -190,13 +202,13 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, "profile": prof.name if prof else None, } finish = dict(prof=prof, no_save=no_save, store=store, html_path=html_path, - as_json=as_json, upload=upload, fail_under=fail_under) + as_json=as_json, upload=upload, fail_under=fail_under, expect=expect) if journeys: report, result, live_mode = _run_journeys( journeys, url, config, latency_wall=latency_wall, error_threshold=error_threshold, timeout=timeout, headers=headers, - replica=bool(app), as_json=as_json) + replica=bool(app), as_json=as_json, combined=not no_combined) _finish(result, report, live_mode=live_mode, **finish) return @@ -259,8 +271,10 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, def _finish(result: dict, report, *, prof, no_save: bool, store: str | None, html_path: str | None, as_json: bool, upload: bool, fail_under: int | None, - live_mode: bool) -> None: + live_mode: bool, expect: int | None = None) -> None: """Save, render, upload and gate a finished run (URLs or journeys alike).""" + if expect is not None: + result["expect"] = expect_block(result, expect) if prof is not None: result["profile"] = scenario_block(prof, report.survives_users) saved_path = None if no_save else write_result(result, store=store) @@ -288,6 +302,50 @@ def _finish(result: dict, report, *, prof, no_save: bool, store: str | None, raise SystemExit(1) +def _meets(survives: int, held_all: bool, users: float) -> bool | None: + """True if it held USERS; None if it held everything tested but we never + tested that many; False if it broke below them.""" + if survives >= users: + return True + return None if held_all else False + + +def expect_block(result: dict, users: int) -> dict: + """Whether the run holds the USERS you expect at peak. + + A URL run compares its verdict. A journey run splits USERS across journeys + by weight: each journey must hold its share on its own, and all journeys + together (if they were ramped together) must hold USERS. It meets the + expectation only if every one of those does.""" + v = result["verdict"] + untested = [j["name"] for j in (result.get("journeys") or []) if j["status"] != "ok"] + journeys = [j for j in (result.get("journeys") or []) if j["status"] == "ok"] + if not journeys: + return {"users": users, "meets": _meets(v["survives_users"], + v["onset_users"] is None, users), + "survives_users": v["survives_users"], "journeys": None, "combined": None} + # A journey that wasn't load-tested still takes its share of the users. + total = sum(j["weight"] for j in result["journeys"]) + per = [] + for j in journeys: + share = users * j["weight"] / total + per.append({"name": j["name"], "users": round(share, 1), + "survives_users": j["survives_users"], + "meets": _meets(j["survives_users"], j["onset_users"] is None, share)}) + checks = [p["meets"] for p in per] + combined = None + if result.get("combined"): + c = result["combined"] + combined = {"survives_users": c["survives_users"], + "meets": _meets(c["survives_users"], c["onset_users"] is None, users)} + checks.append(combined["meets"]) + if untested: + checks.append(None) # their share was never tested + meets = False if False in checks else (None if None in checks else True) + return {"users": users, "meets": meets, "survives_users": v["survives_users"], + "journeys": per, "combined": combined, "untested": untested} + + def _journeys_to_run(directory: Path, only: tuple[str, ...], *, urls_only: bool, url_mode: bool) -> list[Journey]: """The journeys this run replays: none in URL mode (--urls, --path, @@ -317,7 +375,7 @@ def _journeys_to_run(directory: Path, only: tuple[str, ...], *, urls_only: bool, def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wall: float, error_threshold: float, timeout: float, headers: dict[str, str], - replica: bool, as_json: bool) -> tuple: + replica: bool, as_json: bool, combined: bool = True) -> tuple: """The isolation pass: check each journey once, then ramp each one that replays on its own. Returns (report, result, live_mode) for the first journey to break, with every journey's verdict in result["journeys"].""" @@ -372,6 +430,31 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa result["verdict"]["culprit_journey"] = (worst.journey.name if worst.report.onset_users is not None else None) result["journeys"] = journeys_block(runs) + + ramped = [r for r in runs if r.report is not None] + if combined and len(ramped) > 1: + if not as_json: + mix = ", ".join(f"{escape(r.journey.name)} {r.journey.weight:g}" for r in ramped) + console.print(f"\n[bold]All journeys together[/bold] [dim](by weight: {mix})[/dim]", + highlight=False) + progress = ramp_progress(console, latency_wall=latency_wall, as_json=as_json) + live_mode = isinstance(progress, LiveRamp) + try: + with progress: + stages, together_warning = asyncio.run(ramp( + [r.journey for r in ramped], url, levels=levels, + stage_seconds=config["stage_seconds"], timeout=timeout, + max_rps=config["max_rps"], warmup=config["warmup"], + repeat=config["repeat"], headers=headers, think=config["think_time_s"], + progress_cb=progress.starting, on_stage=progress.finished)) + except LoadError as exc: + out.print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) + together = analyze(stages, latency_wall=latency_wall, error_threshold=error_threshold, + rate_capped=config["max_rps"] is not None) + result["combined"] = combined_block(together, runs) + if together_warning and together_warning not in (result["warning"] or ""): + result["warning"] = " ".join(filter(None, [result["warning"], together_warning])) return worst.report, result, live_mode diff --git a/cli/src/prescale_cli/journeys/load.py b/cli/src/prescale_cli/journeys/load.py index d25554a..9cd24a8 100644 --- a/cli/src/prescale_cli/journeys/load.py +++ b/cli/src/prescale_cli/journeys/load.py @@ -1,10 +1,14 @@ -"""Load-test journeys: the isolation pass. +"""Load-test journeys: the isolation pass, then the combined pass. Each journey is first replayed once (`check`). A journey that fails there is stale or unreachable and isn't ramped: a failure with no load says nothing about capacity. Every journey that replays is then ramped on its own, so its verdict (how many users it survives, the step that breaks first, why) doesn't depend on any traffic mix. The run's overall verdict is the first journey to break. + +The combined pass then ramps all of them together, the users split between +journeys by `weight`, to catch interference: journeys competing for something +they share (a database pool, CPU, a cache), which no journey shows on its own. """ from __future__ import annotations @@ -14,7 +18,13 @@ import httpx from prescale_cli.journeys.model import Journey -from prescale_cli.journeys.replay import JourneyCheck, client_options, journey_vu +from prescale_cli.journeys.replay import ( + SEP, + JourneyCheck, + apportion, + client_options, + journey_vu, +) from prescale_cli.loadtest import RunReport, StageResult, run_loadtest @@ -32,14 +42,16 @@ def status(self) -> str: return "stale" if self.check.stale else "unreachable" -async def ramp(journey: Journey, base_url: str, *, levels: list[int], stage_seconds: float, +async def ramp(journey: Journey | list[Journey], base_url: str, *, levels: list[int], + stage_seconds: float, timeout: float = 10.0, max_rps: float | None = None, warmup: bool = True, repeat: int = 1, headers: dict[str, str] | None = None, env: dict[str, str] | None = None, think: float = 0.0, progress_cb=None, on_stage=None, transport: httpx.AsyncBaseTransport | None = None, ) -> tuple[list[StageResult], str | None]: - """Ramp virtual users who each walk JOURNEY, through LEVELS. THINK is the - pause for steps without their own `think`.""" + """Ramp virtual users who each walk JOURNEY, through LEVELS; given a list, + each walk picks one by weight (the combined pass). THINK is the pause for + steps without their own `think`.""" return await run_loadtest( [base_url], levels=levels, stage_seconds=stage_seconds, timeout=timeout, max_rps=max_rps, warmup=warmup, repeat=repeat, headers=headers, @@ -122,3 +134,55 @@ def _step_stats(index: int, step, stage: StageResult | None) -> dict: "p95_ms": round(stat.pct(0.95) * 1000) if has_latency else None, "error_rate": round(stat.error_rate, 4), } + + +def combined_block(report: RunReport, runs: list[JourneyRun]) -> dict: + """The Result's `combined` field: the journeys ramped together by weight. + Users here are the total across journeys, split between them by `apportion`, + so the number on each journey at any level is exact. If the journey that + breaks first does so with no more of its own users than it surely held alone + (the low end of its band), something it shares with the others is the limit: + interference.""" + ramped = [r for r in runs if r.report is not None] + weights = [r.journey.weight for r in ramped] + total = sum(weights) + share = {r.journey.name: r.journey.weight / total for r in ramped} + block: dict = { + "journeys": [r.journey.name for r in ramped], + "mix": {name: round(part * 100, 1) for name, part in share.items()}, + "survives_users": report.survives_users, + "max_tested": report.max_tested, + "onset_users": report.onset_users, + "onset_reason": report.onset_reason, + "confidence": {"survives_low": report.survives_low, + "survives_high": report.survives_high, "stable": report.stable}, + "culprit_journey": None, + "culprit_step": None, + "bottleneck": report.bottleneck, + "interference": None, + } + if report.onset_users is None or not report.culprit_route or SEP not in report.culprit_route: + return block + name, label = report.culprit_route.split(SEP, 1) + run = next((r for r in ramped if r.journey.name == name), None) + if run is None: + return block + index = next((i for i, s in enumerate(run.journey.steps, 1) if s.label == label), None) + block["culprit_journey"] = name + if index is not None: + block["culprit_step"] = {"index": index, "name": run.journey.steps[index - 1].name, + "route": label} + users_of_it = apportion(report.onset_users, weights)[ramped.index(run)] + alone = run.report + if alone.onset_users is None: + held = alone.max_tested + else: + held = alone.survives_low if alone.survives_low is not None else alone.survives_users + if 0 < users_of_it <= held: + block["interference"] = { + "journey": name, + "users_at_break": users_of_it, + "held_alone": held, + "with": [r.journey.name for r in ramped if r.journey.name != name], + } + return block diff --git a/cli/src/prescale_cli/journeys/replay.py b/cli/src/prescale_cli/journeys/replay.py index 4760ff3..633f226 100644 --- a/cli/src/prescale_cli/journeys/replay.py +++ b/cli/src/prescale_cli/journeys/replay.py @@ -29,6 +29,8 @@ from prescale_cli.loadtest import client_headers _SNIPPET = 160 +# Separates journey and step in labels when several journeys run together. +SEP = " › " _MAX_REDIRECTS = 10 @@ -118,18 +120,43 @@ async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, return result -def journey_vu(journey: Journey, base_url: str, env: dict[str, str] | None = None, - think: float = 0.0): - """JOURNEY as a load-test virtual user: walk its steps, pausing for each - step's think time, and start again with a fresh identity until the stage - deadline. A failed step ends that walk (after its pause), like a real user - giving up. Each step is recorded under its label, so the report can name the - step that broke. THINK is the pause for steps that don't set their own - (`--think-time`, or a launch profile's).""" +def apportion(users: int, weights: list[float]) -> list[int]: + """Split USERS into whole numbers proportional to WEIGHTS (largest remainder), + so the counts always add up to USERS.""" + total = sum(weights) + exact = [users * w / total for w in weights] + counts = [int(x) for x in exact] + by_remainder = sorted(range(len(weights)), key=lambda i: exact[i] - counts[i], reverse=True) + for i in by_remainder[:users - sum(counts)]: + counts[i] += 1 + return counts + + +def journey_vu(journeys: Journey | list[Journey], base_url: str, + env: dict[str, str] | None = None, think: float = 0.0): + """A load-test virtual user: walk a journey's steps, pausing for each step's + think time, and start again with a fresh identity until the stage deadline. + A failed step ends that walk (after its pause), like a real user giving up. + + Given several JOURNEYS, the stage's users are split between them by `weight` + (see `apportion`) and each user stays on its journey for the whole stage, so + a journey's weight is its share of the users in the app at once, whatever + its walks take. Steps are then labelled `journey › step`. Each step is + recorded under its label, so the report can name the step that broke. THINK + is the pause for steps that don't set their own (`--think-time`, or a launch + profile's).""" env = dict(os.environ) if env is None else env + mix = [journeys] if isinstance(journeys, Journey) else list(journeys) + weights = [j.weight for j in mix] - async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> None: + async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None, + index: int = 0, users: int = 1) -> None: loop = asyncio.get_running_loop() + journey = mix[0] + if len(mix) > 1: + seats = [j for j, n in zip(mix, apportion(users, weights)) for _ in range(n)] + journey = seats[index] + prefix = f"{journey.name}{SEP}" if len(mix) > 1 else "" while loop.time() < deadline: variables: dict[str, Any] = fresh_uniques() jar = httpx.Cookies() @@ -139,7 +166,7 @@ async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> Non if loop.time() >= deadline: return outcome = await run_step(client, step, base_url, variables, env, jar) - _record(sink, outcome) + _record(sink, outcome, prefix) # A user pauses after a step either way; restarting the instant a step # fails would hammer the app harder than real users who hit an error. pause = step.think or think @@ -156,10 +183,10 @@ async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> Non return vu -def _record(sink, outcome: StepCheck) -> None: +def _record(sink, outcome: StepCheck, prefix: str = "") -> None: """One step's outcome into the stage's numbers. An answer the journey didn't expect counts as a failure, even a 4xx: under load that's a step going wrong.""" - label = outcome.step.label + label = prefix + outcome.step.label if outcome.status is None: sink.record_error(label, outcome.kind or "network") elif outcome.ok or outcome.status >= 500 or outcome.status == 429: diff --git a/cli/src/prescale_cli/loadtest.py b/cli/src/prescale_cli/loadtest.py index 131d9f7..adb4305 100644 --- a/cli/src/prescale_cli/loadtest.py +++ b/cli/src/prescale_cli/loadtest.py @@ -394,7 +394,8 @@ async def _run_stage(client: httpx.AsyncClient, targets: list[str], method: str, think_time: float = 0.0, vu=None) -> StageResult: """Hold USERS virtual users for DURATION. By default each fires requests at TARGETS; VU, when given, is the virtual user itself: an async callable - (client, deadline, sink, gate) that records its own requests (journeys).""" + (client, deadline, sink, gate, index, users) that records its own requests + (journeys). INDEX is this user's place among the stage's USERS.""" sink = _Sink(labels=vu is None) cycle = itertools.cycle(targets) # round-robin spreads load evenly across routes loop = asyncio.get_running_loop() @@ -405,7 +406,8 @@ async def _run_stage(client: httpx.AsyncClient, targets: list[str], method: str, watcher = asyncio.create_task(_watch_loop_lag(stop, lags)) try: if vu is not None: - await asyncio.gather(*(vu(client, deadline, sink, gate) for _ in range(users))) + await asyncio.gather(*(vu(client, deadline, sink, gate, i, users) + for i in range(users))) else: await asyncio.gather( *(_worker(client, method, deadline, sink, lambda: next(cycle), gate, think_time) diff --git a/cli/src/prescale_cli/render.py b/cli/src/prescale_cli/render.py index b7dafb3..fe5e821 100644 --- a/cli/src/prescale_cli/render.py +++ b/cli/src/prescale_cli/render.py @@ -129,6 +129,10 @@ def render_terminal(result: dict, *, show_ramp: bool = True) -> None: bar = text_bar(verdict["survives_users"], scale, width=24, marker=prof["peak_users"]) lines.append(f" [{tone}]{bar}[/{tone}] ┆ = launch peak") + exp = result.get("expect") + if exp: + lines.extend(_expect_lines(exp)) + console.print(Panel("\n".join(lines), title="📈 Readiness report", border_style=color)) if multi and stages: @@ -137,6 +141,29 @@ def render_terminal(result: dict, *, show_ramp: bool = True) -> None: render_journeys(result) +def _expect_lines(exp: dict) -> list[str]: + """The readiness lines for --expect: the answer first, then what decides it.""" + icon = {True: "✅", False: "🛑", None: "❔"}[exp["meets"]] + if not exp.get("journeys"): + tail = {True: f"survives ~{exp['survives_users']}.", + False: f"survives ~{exp['survives_users']}, short by " + f"~{exp['users'] - exp['survives_users']}.", + None: f"held all ~{exp['survives_users']} tested; raise --max-users to know."} + return [f"Expected {icon} ~{exp['users']} users: {tail[exp['meets']]}"] + lines = [f"Expected {icon} ~{exp['users']} users, split across journeys by weight:"] + for j in exp["journeys"]: + mark = {True: "✅", False: "🛑", None: "❔"}[j["meets"]] + lines.append(f" {mark} {escape(j['name'])}: ~{j['users']:g} users, holds " + f"~{j['survives_users']} alone") + c = exp.get("combined") + if c: + mark = {True: "✅", False: "🛑", None: "❔"}[c["meets"]] + lines.append(f" {mark} all together: holds ~{c['survives_users']}") + for name in exp.get("untested") or []: + lines.append(f" ❔ {escape(name)}: not load-tested (stale or unreachable)") + return lines + + def render_journeys(result: dict) -> None: """One row per journey: how many users it survives and the step that breaks first. Each was ramped on its own, so no row depends on the others.""" @@ -168,6 +195,30 @@ def render_journeys(result: dict) -> None: style = "bold red" if j["name"] == first and j["onset_users"] is not None else None table.add_row(name, survives, where, escape(j.get("bottleneck") or ""), style=style) console.print(table) + if result.get("combined"): + _render_combined(result["combined"]) + + +def _render_combined(c: dict) -> None: + """All journeys ramped together by weight, and whether one starved another.""" + mix = ", ".join(f"{escape(name)} {pct:g}%" for name, pct in c["mix"].items()) + lines = [f"[bold]Mix[/bold] {mix}"] + if c["onset_users"] is None: + lines.append(f"Held through {c['max_tested']} users in total.") + else: + step = c.get("culprit_step") + where = (f"{escape(c['culprit_journey'])} › {step['index']}. {escape(step['name'])}" + if c.get("culprit_journey") and step else "—") + lines.append(f"Survives ~{c['survives_users']} users in total; breaks first at {where}.") + inter = c.get("interference") + if inter: + others = ", ".join(escape(n) for n in inter["with"]) + lines.append( + f"[yellow]Interference[/yellow] {escape(inter['journey'])} broke with " + f"{inter['users_at_break']} of its own users, but held {inter['held_alone']} " + f"alone. It competes with {others} for something they share (a database pool, " + "CPU, a cache).") + console.print(Panel("\n".join(lines), title="All journeys together", border_style="cyan")) def _render_routes(result: dict) -> None: diff --git a/cli/src/prescale_cli/schema/result.schema.json b/cli/src/prescale_cli/schema/result.schema.json index e2e406e..7751c93 100644 --- a/cli/src/prescale_cli/schema/result.schema.json +++ b/cli/src/prescale_cli/schema/result.schema.json @@ -98,6 +98,44 @@ "description": "Present on journey runs: one entry per journey. Each journey that replays is ramped on its own; a stale or unreachable one isn't load-tested and carries the reason.", "items": { "$ref": "#/$defs/journey" } }, + "combined": { + "type": "object", + "description": "Journey runs with two or more journeys that replay: all of them ramped together, each user picking a journey by weight. Users are the total across journeys.", + "properties": { + "journeys": { "type": "array", "items": { "type": "string" } }, + "mix": { "type": "object", "additionalProperties": { "type": "number" }, "description": "Percent of the concurrent users on each journey (by weight; each user stays on one journey)." }, + "survives_users": { "type": "integer" }, + "max_tested": { "type": "integer" }, + "onset_users": { "type": ["integer", "null"] }, + "onset_reason": { "type": ["string", "null"], "enum": ["errors", "latency", null] }, + "confidence": { "type": "object" }, + "culprit_journey": { "type": ["string", "null"] }, + "culprit_step": { "type": ["object", "null"] }, + "bottleneck": { "type": ["string", "null"] }, + "interference": { + "type": ["object", "null"], + "description": "Set when the journey that breaks first does so with fewer of its own users than it held alone: it competes with the others for something they share.", + "properties": { + "journey": { "type": "string" }, + "users_at_break": { "type": "integer", "description": "Its own users (exact) at the level where the combined run broke." }, + "held_alone": { "type": "integer" }, + "with": { "type": "array", "items": { "type": "string" } } + } + } + } + }, + "expect": { + "type": "object", + "description": "Present with `run --expect N`: whether the run holds N users at peak. In a journey run N is split across journeys by weight; each journey must hold its share alone, and the combined run must hold N.", + "properties": { + "users": { "type": "integer" }, + "meets": { "type": ["boolean", "null"], "description": "null: held every level tested, but fewer users than expected were tested." }, + "survives_users": { "type": "integer" }, + "journeys": { "type": ["array", "null"] }, + "combined": { "type": ["object", "null"] }, + "untested": { "type": "array", "items": { "type": "string" }, "description": "Journeys that weren't load-tested; their share of N is unknown." } + } + }, "investigation": { "type": "object", "description": "Present when `prescale investigate` diagnosed a failure.", diff --git a/cli/tests/test_journeys_load.py b/cli/tests/test_journeys_load.py index 6d90c15..fecb4fb 100644 --- a/cli/tests/test_journeys_load.py +++ b/cli/tests/test_journeys_load.py @@ -220,7 +220,9 @@ async def fake_check_all(journeys, url, *, timeout, headers): return out async def fake_ramp(journey, url, *, levels, **kwargs): - routes = {s.label: RouteStat(total=10, latencies=[0.01] * 10) for s in journey.steps} + mix = journey if isinstance(journey, list) else [journey] + routes = {(f"{j.name} › " if len(mix) > 1 else "") + s.label: + RouteStat(total=10, latencies=[0.01] * 10) for j in mix for s in j.steps} return [StageResult(users=u, duration=1.0, routes=dict(routes)) for u in levels], None monkeypatch.setattr(run_cmd, "_check_all", fake_check_all) @@ -258,3 +260,117 @@ async def spy(journey, url, **kwargs): "--dir", _dir(tmp_path, BROWSE)]) assert res.exit_code == 0, res.output assert seen == {"browse": 1.5} + + +# --- the combined pass and --expect ------------------------------------------------ + +def test_apportion_adds_up_and_follows_weights(): + from prescale_cli.journeys.replay import apportion + assert apportion(10, [9, 1]) == [9, 1] + assert apportion(30, [1, 1]) == [15, 15] + assert apportion(3, [60, 25, 8, 5, 2]) == [2, 1, 0, 0, 0] + assert sum(apportion(97, [60, 25, 8, 5, 2])) == 97 + + +def test_mixed_users_stay_on_their_journey_whatever_its_walk_takes(): + from prescale_cli.journeys.replay import journey_vu + from prescale_cli.loadtest import _Sink + + quick = parse({**BROWSE, "name": "quick", "weight": 1}) + slow = parse({**BROWSE, "name": "slow", "weight": 1, + "steps": [{"request": {"path": "/slow"}, "think": "1s"}]}) + vu = journey_vu([quick, slow], "http://shop.test", env={}) + sinks = [_Sink(labels=False) for _ in range(10)] + + async def go(): + transport = httpx.MockTransport(lambda r: httpx.Response(200)) + async with httpx.AsyncClient(transport=transport) as c: + deadline = asyncio.get_running_loop().time() + 0.3 + await asyncio.gather(*(vu(c, deadline, sinks[i], None, i, 10) for i in range(10))) + asyncio.run(go()) + # each user only ever walked one journey; half the users are on each + walked = [set(s.routes) for s in sinks] + assert all(len(w) == 1 for w in walked) + assert sum(w == {"quick › GET /"} for w in walked) == 5 + assert sum(w == {"slow › GET /slow"} for w in walked) == 5 + + +def _report(survives, onset, culprit=None, max_tested=100): + from prescale_cli.loadtest import RunReport + return RunReport(stages=[], survives_users=survives, max_tested=max_tested, + onset_users=onset, onset_reason="errors" if onset else None, + culprit_route=culprit, survives_low=survives, survives_high=survives) + + +def test_combined_block_flags_interference(): + from prescale_cli.journeys.load import combined_block + checkout = parse({**SHOP, "weight": 1}) + search = parse({**BROWSE, "name": "search", "weight": 3}) + runs = [JourneyRun(checkout, None, report=_report(40, 50, "POST /api/orders")), + JourneyRun(search, None, report=_report(100, None))] + # together: breaks at 60 users in total, at checkout's order step. + # checkout had only 60 * 1/4 = 15 of its own users there, but held 40 alone. + block = combined_block(_report(50, 60, "checkout › POST /api/orders"), runs) + assert block["mix"] == {"checkout": 25.0, "search": 75.0} + assert block["culprit_journey"] == "checkout" + assert block["culprit_step"] == {"index": 2, "name": "place order", + "route": "POST /api/orders"} + assert block["interference"] == {"journey": "checkout", "users_at_break": 15.0, + "held_alone": 40, "with": ["search"]} + # breaking with more of its own users than it held alone is not interference + calm = combined_block(_report(150, 200, "checkout › POST /api/orders"), runs) + assert calm["interference"] is None + held = combined_block(_report(100, None), runs) + assert held["culprit_journey"] is None and held["interference"] is None + + +def test_expect_splits_users_across_journeys_by_weight(): + from prescale_cli.commands.run import expect_block + result = { + "verdict": {"survives_users": 20, "onset_users": 30}, + "journeys": [ + {"name": "browse", "status": "ok", "weight": 60, "survives_users": 400, + "onset_users": None}, + {"name": "checkout", "status": "ok", "weight": 5, "survives_users": 20, + "onset_users": 30}, + {"name": "old", "status": "stale", "weight": 5}, + ], + "combined": {"survives_users": 400, "onset_users": None}, + } + exp = expect_block(result, 300) + assert exp["meets"] is False + # the stale journey still takes its share of the users (weights total 70) + assert [(j["name"], j["users"], j["meets"]) for j in exp["journeys"]] == [ + ("browse", 257.1, True), ("checkout", 21.4, False)] + assert exp["combined"] == {"survives_users": 400, "meets": True} + assert exp["untested"] == ["old"] + # every tested share fits, but "old" was never tested: unknown, not a yes + assert expect_block(result, 100)["meets"] is None + fresh = {**result, "journeys": result["journeys"][:2]} + assert expect_block(fresh, 100)["meets"] is True + # more users than were ever tested on a journey that never broke: unknown + assert expect_block(fresh, 6000)["journeys"][0]["meets"] is None + + +def test_expect_on_a_url_run(): + from prescale_cli.commands.run import expect_block + broke = {"verdict": {"survives_users": 90, "onset_users": 100}} + assert expect_block(broke, 50)["meets"] is True + assert expect_block(broke, 200)["meets"] is False + held = {"verdict": {"survives_users": 200, "onset_users": None}} + assert expect_block(held, 500)["meets"] is None + + +def test_run_combined_pass_and_expect_end_to_end(tmp_path, monkeypatch): + _fake_checks(monkeypatch) + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--json", "--no-save", + "-u", "5", "--expect", "4", + "--dir", _dir(tmp_path, SHOP, BROWSE)]) + assert res.exit_code == 0, res.output + result = json.loads(res.stdout) + assert result["combined"]["mix"] == {"browse": 92.3, "checkout": 7.7} + assert result["expect"]["meets"] is True + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--json", "--no-save", + "-u", "5", "--no-combined", + "--dir", _dir(tmp_path, SHOP, BROWSE)]) + assert "combined" not in json.loads(res.stdout)