From 543652d0545c229e7872e51caa29861fe0d96f69 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:02:41 +0000 Subject: [PATCH 01/11] feat(journeys): journey file format and prescale journeys verify --- CHANGELOG.md | 13 + README.md | 39 +++ cli/src/prescale_cli/commands/journeys.py | 164 +++++++++++ cli/src/prescale_cli/journeys/__init__.py | 6 + cli/src/prescale_cli/journeys/model.py | 332 ++++++++++++++++++++++ cli/src/prescale_cli/journeys/replay.py | 133 +++++++++ cli/src/prescale_cli/main.py | 3 + cli/tests/test_journeys.py | 260 +++++++++++++++++ examples/README.md | 9 + examples/fragile-shop.py | 71 ++++- examples/journeys/browse.yaml | 12 + examples/journeys/checkout.yaml | 25 ++ 12 files changed, 1064 insertions(+), 3 deletions(-) create mode 100644 cli/src/prescale_cli/commands/journeys.py create mode 100644 cli/src/prescale_cli/journeys/__init__.py create mode 100644 cli/src/prescale_cli/journeys/model.py create mode 100644 cli/src/prescale_cli/journeys/replay.py create mode 100644 cli/tests/test_journeys.py create mode 100644 examples/journeys/browse.yaml create mode 100644 examples/journeys/checkout.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0240859..e2909bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +### Added +- **Journeys: your users' real paths, saved in your repo.** A journey file + (`prescale/journeys/.yaml`) lists the requests a user's path makes (log in, add to + cart, place an order), carrying tokens and IDs from one step to the next. + `prescale journeys` lists them; `prescale journeys verify ` replays each once, as one + user, and names the step that no longer answers as expected (the journey is stale, or the + app is broken). It exits non-zero, so it works as a CI check. + - Secrets come from environment variables (`{{env.NAME}}`); a literal password, token or + API key in a journey file is refused. + - Journeys that change data only run against local hosts or hosts you allow with + `--allow-writes`; journeys that send messages or delete data are never replayed. + - The example shop has a JSON API and two example journeys in `examples/journeys/`. + ### Fixed - **`prescale mcp` works on a fresh install again.** `pip install 'prescale[mcp]'` picked up the new mcp 2.x, which renamed the server class PreScale uses, so the MCP server wouldn't start and told diff --git a/README.md b/README.md index 178406b..552c8a1 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ From source: `git clone https://github.com/pyjeebz/PreScale.git && pip install . | [`run `](#run--what-breaks-first) | Ramp traffic; report what breaks first and at what load | | [`investigate `](#investigate--and-why) | …and *why* it breaks, plus how to fix it (`--app` to start your app first: [replica mode](#replica-mode--test-your-app-not-your-host)) | | [`audit `](#audit--hygiene-no-load) | Load-free HTTP hygiene check (compression, caching, CDN, HTTP/2) | +| [`journeys`](#journeys--your-users-real-paths) | List your journey files; `journeys verify ` checks they still replay | | [`compare`](#ci--gate-on-capacity) | Capacity diff of two runs; regression gating for CI | | [`profiles`](#frame-it-as-a-launch) | Launch scenarios for `--profile` | | [`history` / `show`](#saved-runs-and-the-json-contract) | List and re-open saved runs | @@ -166,6 +167,44 @@ Try this → Add a pooler (e.g. pgbouncer) and check the pool checkout timeout. ``` +### `journeys` — your users' real paths + +A journey is a path a user takes through your app (log in, add to cart, place an order), saved as the requests it makes. Journeys live in your repo as `prescale/journeys/.yaml`: + +```yaml +version: 1 +name: checkout +type: checkout +steps: + - name: log in + request: + method: POST + path: /api/login + json: {email: "{{unique.email}}", password: "{{env.SHOP_PASSWORD}}"} + expect: {status: 200} + extract: {token: body.token} + - name: place order + request: + method: POST + path: /api/orders + headers: {Authorization: "Bearer {{token}}"} + expect: {status: 201} +``` + +```bash +prescale journeys # list them +prescale journeys verify http://localhost:3000 # replay each once, as one user +``` + +`verify` stops at the first step that doesn't answer as expected and calls the journey **stale**: the app changed, or it's broken. It exits non-zero, so it works as a CI check. + +- `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. +- **Secrets never go in the file.** A literal password, token, API key or `Authorization` value is refused; use `{{env.NAME}}`. +- Paths must start with `/`: a journey only calls the app it's run against. +- Journeys that change data (POST, PUT, PATCH, DELETE) run against local hosts, or a host you name with `--allow-writes staging.myapp.com`. Types `side_effect` (emails, SMS, invites) and `destructive` are never replayed. + +Try it on the example shop: `python examples/fragile-shop.py 8400`, then `SHOP_PASSWORD=demo prescale journeys verify http://localhost:8400 --dir examples/journeys`. + ### `audit` — hygiene, no load A fast, load-free check of the HTTP-level footguns that decide how you scale — compression, static-asset caching, CDN, HTTP version, cookies on assets. Cheap enough to run on every commit. diff --git a/cli/src/prescale_cli/commands/journeys.py b/cli/src/prescale_cli/commands/journeys.py new file mode 100644 index 0000000..626b39a --- /dev/null +++ b/cli/src/prescale_cli/commands/journeys.py @@ -0,0 +1,164 @@ +"""`prescale journeys` - list the app's journeys, and check they still replay. + +Journeys live in the app's repo as `prescale/journeys/*.yaml`. `verify` replays +each one once, as a single user: a step that fails there means the journey is +stale (the app changed) or the app is broken, never that it broke under load. +""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from urllib.parse import urlparse + +import click +from rich.console import Console +from rich.markup import escape +from rich.table import Table + +from prescale_cli.commands.upload import header_option +from prescale_cli.journeys.model import DEFAULT_DIR, Journey, load_dir +from prescale_cli.journeys.replay import JourneyCheck, check + +console = Console() + +_LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"} + +dir_option = click.option( + "--dir", "directory", type=click.Path(file_okay=False, path_type=Path), + default=DEFAULT_DIR, show_default=True, help="Where the journey files are.") + + +@click.group(invoke_without_command=True) +@dir_option +@click.pass_context +def journeys(ctx: click.Context, directory: Path) -> None: + """List the app's journeys (prescale/journeys/*.yaml). + + \b + Check they still replay against your app: + prescale journeys verify http://localhost:3000 + """ + if ctx.invoked_subcommand is not None: + return + found = _load_or_exit(directory) + table = Table(title=f"Journeys in {directory}") + for col in ("Name", "Type", "Weight", "Steps", "Changes data", "File"): + table.add_column(col) + for j in found: + table.add_row(j.name, j.type, f"{j.weight:g}", str(len(j.steps)), + "yes" if j.writes else "no", str(j.source)) + console.print(table) + + +@journeys.command() +@click.argument("url") +@dir_option +@click.option("--journey", "only", multiple=True, + help="Check only this journey (repeatable).") +@click.option("--allow-writes", "allow_writes", multiple=True, metavar="HOST", + help="Let journeys that change data (POST, PUT, PATCH, DELETE) run against " + "this non-local host, e.g. staging.myapp.com (repeatable).") +@click.option("--timeout", default=10.0, type=float, help="Per-request timeout in seconds.") +@header_option +def verify(url: str, directory: Path, only: tuple[str, ...], allow_writes: tuple[str, ...], + timeout: float, headers: dict[str, str]) -> None: + """Replay each journey once, as one user, and report any step that fails. + + Exits non-zero if a journey is stale or the app couldn't be reached. + + \b + prescale journeys verify http://localhost:3000 + prescale journeys verify https://staging.myapp.com --allow-writes staging.myapp.com + """ + parsed = urlparse(url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + console.print(f"[red]Error:[/red] '{url}' is not a URL " + "(expected e.g. http://localhost:3000).") + raise SystemExit(2) + found = _load_or_exit(directory) + if only: + missing = sorted(set(only) - {j.name for j in found}) + if missing: + console.print(f"[red]Error:[/red] no journey named {', '.join(missing)} " + f"in {directory}.") + raise SystemExit(2) + found = [j for j in found if j.name in only] + + host = (parsed.hostname or "").lower() + _refuse_unallowed_writes(found, host, {h.lower() for h in allow_writes}) + unset = sorted({name for j in found for name in j.env_names()} - set(os.environ)) + if unset: + console.print(f"[red]Error:[/red] the journeys read environment variable(s) that " + f"aren't set: {', '.join(unset)}.") + raise SystemExit(2) + + base = url if url.endswith("/") else url + "/" + results = asyncio.run(_check_all(found, base, timeout=timeout, headers=headers)) + for result in results: + _print_check(result) + + broken = [r for r in results if not r.ok] + if broken: + stale = [r.journey.name for r in broken if r.stale] + if stale: + console.print(f"\n[yellow]{len(stale)} stale:[/yellow] {', '.join(stale)}. The app " + "no longer answers the way these journeys expect. If the change was " + "on purpose, update the journey files; if not, it's a bug.") + unreached = [r.journey.name for r in broken if not r.stale] + if unreached: + console.print(f"\n[red]{len(unreached)} couldn't run:[/red] {', '.join(unreached)}.") + raise SystemExit(1) + console.print(f"\n[green]✓ All {len(results)} journeys replay.[/green]") + + +def _load_or_exit(directory: Path) -> list[Journey]: + found, errors = load_dir(directory) + for error in errors: + console.print(f"[red]Error:[/red] {escape(error)}", highlight=False) + if errors: + raise SystemExit(2) + if not found: + console.print(f"No journeys in {directory}. Add one as {directory}/.yaml.") + raise SystemExit(1) + return found + + +def _refuse_unallowed_writes(found: list[Journey], host: str, allowed: set[str]) -> None: + """Journeys that change data run on local hosts, or on hosts named with + --allow-writes. Anywhere else they could change real data.""" + if host in _LOCAL_HOSTS or host in allowed: + return + writers = [j.name for j in found if j.writes] + if writers: + console.print(f"[red]Error:[/red] {', '.join(writers)} change data (POST, PUT, PATCH " + f"or DELETE). Run them against {host} only if it's a staging or test " + f"environment, with --allow-writes {host}.") + raise SystemExit(2) + + +async def _check_all(found: list[Journey], base: str, *, timeout: float, + headers: dict[str, str]) -> list[JourneyCheck]: + return [await check(j, base, timeout=timeout, headers=headers) for j in found] + + +def _print_check(result: JourneyCheck) -> None: + journey = result.journey + mark = "[green]✓[/green]" if result.ok else ("[yellow]stale[/yellow]" if result.stale + else "[red]✗[/red]") + console.print(f"\n{mark} [bold]{journey.name}[/bold] [dim]({journey.type}, " + f"{len(journey.steps)} steps)[/dim]") + for i, step in enumerate(journey.steps, 1): + done = result.steps[i - 1] if i <= len(result.steps) else None + if done is None: + console.print(f" [dim]{i}. {step.name} {step.label} (not run)[/dim]") + elif done.ok: + console.print(f" {i}. {step.name} [dim]{step.label}[/dim] " + f"{done.status} {done.latency * 1000:.0f}ms") + else: + console.print(f" {i}. {step.name} [dim]{step.label}[/dim] " + f"[red]{escape(done.problem or '')}[/red]", highlight=False) + if done.status is not None and done.status >= 500: + console.print(" [dim]A 5xx with no load is usually a bug in the app, " + "not a stale journey.[/dim]") diff --git a/cli/src/prescale_cli/journeys/__init__.py b/cli/src/prescale_cli/journeys/__init__.py new file mode 100644 index 0000000..3dab79d --- /dev/null +++ b/cli/src/prescale_cli/journeys/__init__.py @@ -0,0 +1,6 @@ +"""Journeys: a user's path through the app (log in, search, add to cart, order), +saved as the HTTP requests it makes, so PreScale can replay it. + +A journey lives in the app's own repo as `prescale/journeys/.yaml`. +`model` loads and checks those files; `replay` runs a journey's steps. +""" diff --git a/cli/src/prescale_cli/journeys/model.py b/cli/src/prescale_cli/journeys/model.py new file mode 100644 index 0000000..1ca708b --- /dev/null +++ b/cli/src/prescale_cli/journeys/model.py @@ -0,0 +1,332 @@ +"""The journey file format (version 1), and the templating its steps use. + +A journey file is YAML, one journey per file, committed to the app's repo: + + version: 1 + name: checkout + type: checkout + steps: + - name: log in + request: {method: POST, path: /api/login, + json: {email: "{{unique.email}}", password: "{{env.SHOP_PASSWORD}}"}} + expect: {status: 200} + extract: {token: body.token} + - name: place order + request: {method: POST, path: /api/orders, + headers: {Authorization: "Bearer {{token}}"}} + expect: {status: [200, 201]} + +Placeholders: `{{name}}` is a value an earlier step extracted; `{{unique.email}}` +and `{{unique.id}}` are fresh for each run of the journey; `{{env.NAME}}` reads an +environment variable, which is how secrets stay out of the file. Everything here +is pure: loading never touches the network. +""" + +from __future__ import annotations + +import re +import secrets +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +FORMAT_VERSION = 1 +DEFAULT_DIR = Path("prescale") / "journeys" + +# Journey types PreScale replays, and each one's default share of the traffic mix. +TYPES: dict[str, int] = { + "browse": 60, "search": 25, "account": 8, "checkout": 5, "write": 5, "auth": 2, +} +# Types PreScale never replays: they message real people or destroy data. +REFUSED_TYPES = { + "side_effect": "it sends email, SMS or invites to real people", + "destructive": "it deletes data or changes admin settings", +} +METHODS = {"GET", "HEAD", "OPTIONS", "POST", "PUT", "PATCH", "DELETE"} +READ_METHODS = {"GET", "HEAD", "OPTIONS"} + +_PLACEHOLDER = re.compile(r"\{\{\s*([A-Za-z_][\w.]*)\s*\}\}") +_SENSITIVE_KEY = re.compile(r"pass(word|wd)?|secret|token|api[-_]?key|authorization|cookie", + re.IGNORECASE) +_UNIQUE = {"email", "id"} +_SOURCES = ("body", "header") + + +class JourneyError(Exception): + """A journey file that can't be used, with a message that says how to fix it.""" + + +@dataclass +class Step: + name: str + method: str + path: str + headers: dict[str, str] = field(default_factory=dict) + json: Any = None + body: str | None = None + expect: list[int] | None = None # None: any status below 400 + extract: dict[str, str] = field(default_factory=dict) + think: float = 0.0 + + @property + def label(self) -> str: + """The route as reports show it: method plus path, placeholders kept.""" + return f"{self.method} {self.path.split('?')[0]}" + + def accepts(self, status: int) -> bool: + if self.expect is None: + return status < 400 + return status in self.expect + + +@dataclass +class Journey: + name: str + type: str + weight: float + steps: list[Step] + source: Path | None = None + + @property + def writes(self) -> bool: + """Whether replaying it can change data on the target.""" + return any(step.method not in READ_METHODS for step in self.steps) + + def env_names(self) -> set[str]: + """Environment variables its steps read.""" + return {name[4:] for step in self.steps for name in _names(_step_values(step)) + if name.startswith("env.")} + + +# --- loading -------------------------------------------------------------- + +def load_dir(directory: Path = DEFAULT_DIR) -> tuple[list[Journey], list[str]]: + """Every journey in DIRECTORY, sorted by name, plus one error message per file + that couldn't be loaded. A missing directory is no journeys, not an error.""" + if not directory.is_dir(): + return [], [] + journeys: list[Journey] = [] + errors: list[str] = [] + for path in sorted([*directory.glob("*.yaml"), *directory.glob("*.yml")]): + try: + journeys.append(load_file(path)) + except JourneyError as exc: + errors.append(f"{path}: {exc}") + seen: dict[str, Path] = {} + for journey in journeys: + if journey.name in seen: + errors.append(f"{journey.source}: the name '{journey.name}' is already used by " + f"{seen[journey.name]}") + seen.setdefault(journey.name, journey.source) + return sorted(journeys, key=lambda j: j.name), errors + + +def load_file(path: Path) -> Journey: + try: + data = yaml.safe_load(path.read_text()) + except yaml.YAMLError as exc: + raise JourneyError(f"not valid YAML ({exc})") from exc + journey = parse(data) + journey.source = path + return journey + + +def parse(data: Any) -> Journey: + """Check a decoded journey document and build a `Journey`.""" + if not isinstance(data, dict): + raise JourneyError("a journey file must be a mapping with version, name and steps") + _only(data, {"version", "name", "type", "weight", "steps"}, "the journey") + if data.get("version") != FORMAT_VERSION: + raise JourneyError(f"version must be {FORMAT_VERSION} (got {data.get('version')!r})") + name = data.get("name") + if not isinstance(name, str) or not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", name): + raise JourneyError("name must be lowercase letters, digits, - or _ (e.g. checkout)") + jtype = data.get("type", "browse") + if jtype in REFUSED_TYPES: + raise JourneyError(f"type '{jtype}' is never replayed: {REFUSED_TYPES[jtype]}") + if jtype not in TYPES: + raise JourneyError(f"type must be one of {', '.join(TYPES)} (got {jtype!r})") + weight = data.get("weight", TYPES[jtype]) + if isinstance(weight, bool) or not isinstance(weight, (int, float)) or weight <= 0: + raise JourneyError("weight must be a positive number") + raw_steps = data.get("steps") + if not isinstance(raw_steps, list) or not raw_steps: + raise JourneyError("steps must be a non-empty list") + + steps = [_parse_step(raw, i) for i, raw in enumerate(raw_steps, 1)] + _check_placeholders(steps) + return Journey(name=name, type=jtype, weight=float(weight), steps=steps) + + +def _parse_step(raw: Any, index: int) -> Step: + where = f"step {index}" + if not isinstance(raw, dict): + raise JourneyError(f"{where} must be a mapping with name and request") + _only(raw, {"name", "request", "expect", "extract", "think"}, where) + name = raw.get("name") or f"step {index}" + if not isinstance(name, str): + raise JourneyError(f"{where}: name must be text") + where = f"step {index} ({name})" + + request = raw.get("request") + if not isinstance(request, dict): + raise JourneyError(f"{where}: request must be a mapping with method and path") + _only(request, {"method", "path", "headers", "json", "body"}, f"{where} request") + method = str(request.get("method", "GET")).upper() + if method not in METHODS: + raise JourneyError(f"{where}: method {method} isn't supported") + path = request.get("path") + if not isinstance(path, str) or not path.startswith("/"): + raise JourneyError(f"{where}: path must start with / (journeys only call the app " + "they're run against, never another site)") + headers = request.get("headers") or {} + if not isinstance(headers, dict) or not all(isinstance(v, str) for v in headers.values()): + raise JourneyError(f"{where}: headers must map names to text") + if "json" in request and "body" in request: + raise JourneyError(f"{where}: give either json or body, not both") + body = request.get("body") + if body is not None and not isinstance(body, str): + raise JourneyError(f"{where}: body must be text (use json for structured data)") + _check_secrets({"headers": headers, "json": request.get("json")}, where) + + return Step( + name=name, method=method, path=path, + headers={str(k): v for k, v in headers.items()}, + json=request.get("json"), body=body, + expect=_parse_expect(raw.get("expect"), where), + extract=_parse_extract(raw.get("extract"), where), + think=_parse_seconds(raw.get("think", 0), where), + ) + + +def _parse_expect(raw: Any, where: str) -> list[int] | None: + if raw is None: + return None + if not isinstance(raw, dict) or set(raw) != {"status"}: + raise JourneyError(f"{where}: expect must be {{status: 200}} or {{status: [200, 201]}}") + status = raw["status"] + codes = status if isinstance(status, list) else [status] + if not codes or not all(isinstance(c, int) and 100 <= c <= 599 for c in codes): + raise JourneyError(f"{where}: expect status must be HTTP status codes") + return codes + + +def _parse_extract(raw: Any, where: str) -> dict[str, str]: + if raw is None: + return {} + if not isinstance(raw, dict): + raise JourneyError(f"{where}: extract must map names to body. or header.") + for name, source in raw.items(): + if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z_]\w*", name) \ + or name in {"env", "unique"}: + raise JourneyError(f"{where}: can't extract into {name!r}") + if not isinstance(source, str) or source.split(".", 1)[0] not in _SOURCES \ + or "." not in source: + raise JourneyError(f"{where}: extract {name} must come from body. " + "or header.") + return dict(raw) + + +def _parse_seconds(raw: Any, where: str) -> float: + if isinstance(raw, bool): + raise JourneyError(f"{where}: think must be seconds, e.g. 2 or 2s or 500ms") + if isinstance(raw, (int, float)) and raw >= 0: + return float(raw) + match = re.fullmatch(r"(\d+(?:\.\d+)?)(ms|s)", str(raw).strip()) + if not match: + raise JourneyError(f"{where}: think must be seconds, e.g. 2 or 2s or 500ms") + value = float(match.group(1)) + return value / 1000 if match.group(2) == "ms" else value + + +def _only(data: dict, allowed: set[str], where: str) -> None: + unknown = sorted(set(data) - allowed) + if unknown: + raise JourneyError(f"{where} has unknown field(s): {', '.join(map(str, unknown))}") + + +def _check_secrets(value: Any, where: str, key: str = "") -> None: + """Refuse a literal value under a secret-looking key: secrets come from {{env.*}} + (or an earlier step's extract), so they never end up committed in the file.""" + if isinstance(value, dict): + for k, v in value.items(): + _check_secrets(v, where, str(k)) + elif isinstance(value, list): + for v in value: + _check_secrets(v, where, key) + elif isinstance(value, str) and key and _SENSITIVE_KEY.search(key) \ + and not _PLACEHOLDER.search(value): + raise JourneyError(f"{where}: {key} has a literal value. Put secrets in an " + f"environment variable and write \"{{{{env.NAME}}}}\" instead") + + +def _check_placeholders(steps: list[Step]) -> None: + """Every {{name}} must be extracted by an earlier step, or be env./unique.""" + known: set[str] = set() + for i, step in enumerate(steps, 1): + for name in _names(_step_values(step)): + if name.startswith("env."): + continue + if name.startswith("unique."): + if name[7:] not in _UNIQUE: + raise JourneyError(f"step {i} ({step.name}): {{{{{name}}}}} isn't known; " + "use unique.email or unique.id") + continue + if name not in known: + raise JourneyError(f"step {i} ({step.name}) uses {{{{{name}}}}}, but no " + "earlier step extracts it") + known.update(step.extract) + + +def _step_values(step: Step) -> list[Any]: + return [step.path, step.headers, step.json, step.body] + + +def _names(value: Any) -> list[str]: + if isinstance(value, str): + return _PLACEHOLDER.findall(value) + if isinstance(value, dict): + return [n for k, v in value.items() for n in (*_names(k), *_names(v))] + if isinstance(value, list): + return [n for v in value for n in _names(v)] + return [] + + +# --- templating ----------------------------------------------------------- + +class TemplateError(Exception): + """A placeholder with no value at replay time (e.g. an unset env variable).""" + + +def fresh_uniques() -> dict[str, str]: + """Values for {{unique.*}}, new for each run of a journey.""" + token = secrets.token_hex(6) + # example.com is reserved for examples (RFC 2606): mail to it goes nowhere. + return {"unique.email": f"prescale+{token}@example.com", "unique.id": token} + + +def render(value: Any, variables: dict[str, Any], env: dict[str, str]) -> Any: + """Fill placeholders in a step value. A string that is exactly one placeholder + keeps the value's type (an extracted id stays a number in JSON).""" + if isinstance(value, str): + whole = _PLACEHOLDER.fullmatch(value.strip()) + if whole: + return _lookup(whole.group(1), variables, env) + return _PLACEHOLDER.sub(lambda m: str(_lookup(m.group(1), variables, env)), value) + if isinstance(value, dict): + return {render(k, variables, env): render(v, variables, env) for k, v in value.items()} + if isinstance(value, list): + return [render(v, variables, env) for v in value] + return value + + +def _lookup(name: str, variables: dict[str, Any], env: dict[str, str]) -> Any: + if name.startswith("env."): + if name[4:] not in env: + raise TemplateError(f"environment variable {name[4:]} isn't set") + return env[name[4:]] + if name not in variables: + raise TemplateError(f"{{{{{name}}}}} has no value") + return variables[name] diff --git a/cli/src/prescale_cli/journeys/replay.py b/cli/src/prescale_cli/journeys/replay.py new file mode 100644 index 0000000..b6e67ba --- /dev/null +++ b/cli/src/prescale_cli/journeys/replay.py @@ -0,0 +1,133 @@ +"""Run a journey's steps against an app. + +`check` replays a journey once, as a single user, to see whether it still works +against the app as it is now. A step that fails its expectation here, with no +load at all, means the journey is stale (the app changed), not that the app +broke under load. Each run gets its own client, so cookies and extracted values +never leak between users. +""" + +from __future__ import annotations + +import os +import time +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import urljoin + +import httpx + +from prescale_cli.journeys.model import Journey, Step, TemplateError, fresh_uniques, render +from prescale_cli.loadtest import client_headers + +_SNIPPET = 160 + + +@dataclass +class StepCheck: + step: Step + status: int | None = None + latency: float = 0.0 + ok: bool = False + problem: str | None = None # why it failed, in plain words + + +@dataclass +class JourneyCheck: + journey: Journey + steps: list[StepCheck] = field(default_factory=list) + + @property + def ok(self) -> bool: + return len(self.steps) == len(self.journey.steps) and all(s.ok for s in self.steps) + + @property + def failed(self) -> StepCheck | None: + return next((s for s in self.steps if not s.ok), None) + + @property + def stale(self) -> bool: + """The app answered, but not the way the journey expects.""" + failed = self.failed + return failed is not None and failed.status is not None + + +async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, + headers: dict[str, str] | None = None, env: dict[str, str] | None = None, + transport: httpx.AsyncBaseTransport | None = None) -> JourneyCheck: + """Replay JOURNEY once against BASE_URL and stop at the first step that fails.""" + env = dict(os.environ) if env is None else env + result = JourneyCheck(journey) + variables: dict[str, Any] = fresh_uniques() + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True, + headers=client_headers(headers), transport=transport) as client: + for step in journey.steps: + outcome = await run_step(client, step, base_url, variables, env) + result.steps.append(outcome) + if not outcome.ok: + break + return result + + +async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, + variables: dict[str, Any], env: dict[str, str]) -> StepCheck: + """Send one step, judge it against `expect`, and pull its `extract` values + into VARIABLES for the steps after it.""" + outcome = StepCheck(step) + try: + path = render(step.path, variables, env) + req_headers = render(step.headers, variables, env) + payload = render(step.json, variables, env) if step.json is not None else None + body = render(step.body, variables, env) if step.body is not None else None + except TemplateError as exc: + outcome.problem = str(exc) + return outcome + + start = time.perf_counter() + try: + resp = await client.request(step.method, urljoin(base_url, path), + headers=req_headers, json=payload, content=body) + except httpx.TimeoutException: + outcome.problem = "timed out" + return outcome + except httpx.HTTPError as exc: + outcome.problem = f"couldn't reach the app ({exc.__class__.__name__})" + return outcome + outcome.latency = time.perf_counter() - start + outcome.status = resp.status_code + + if not step.accepts(resp.status_code): + wanted = (" or ".join(map(str, step.expect)) if step.expect else "a status below 400") + outcome.problem = f"returned {resp.status_code}, expected {wanted}" + snippet = resp.text.strip().replace("\n", " ")[:_SNIPPET] + if snippet: + outcome.problem += f": {snippet}" + return outcome + + for name, source in step.extract.items(): + value = extract(resp, source) + if value is None: + outcome.problem = f"couldn't find {source} in the response to extract {name}" + return outcome + variables[name] = value + outcome.ok = True + return outcome + + +def extract(resp: httpx.Response, source: str) -> Any: + """Read `body.a.b.0` (JSON) or `header.Name` from a response; None if missing.""" + kind, _, where = source.partition(".") + if kind == "header": + return resp.headers.get(where) + try: + value: Any = resp.json() + except ValueError: + return None + for part in where.split("."): + if isinstance(value, dict) and part in value: + value = value[part] + elif isinstance(value, list) and part.isdigit() and int(part) < len(value): + value = value[int(part)] + else: + return None + return value diff --git a/cli/src/prescale_cli/main.py b/cli/src/prescale_cli/main.py index d87c4c5..67f66a8 100644 --- a/cli/src/prescale_cli/main.py +++ b/cli/src/prescale_cli/main.py @@ -9,6 +9,7 @@ compare, history, investigate, + journeys, login, mcp, profiles, @@ -33,6 +34,7 @@ def cli() -> None: prescale run http://localhost:8000 # load test: what breaks first prescale investigate http://localhost:8000 # ...and why, plus how to fix it prescale audit https://myapp.com # static scaling-hygiene check + prescale journeys verify http://localhost:8000 # check your journeys still replay prescale history # list saved runs prescale compare # capacity diff of the last two runs prescale profiles # launch scenarios for --profile @@ -45,6 +47,7 @@ def cli() -> None: cli.add_command(run.run) cli.add_command(investigate.investigate) +cli.add_command(journeys.journeys) cli.add_command(audit.audit) cli.add_command(history.history) cli.add_command(compare.compare) diff --git a/cli/tests/test_journeys.py b/cli/tests/test_journeys.py new file mode 100644 index 0000000..68c8142 --- /dev/null +++ b/cli/tests/test_journeys.py @@ -0,0 +1,260 @@ +"""Journey files: the format, its safety rules, and replaying a journey once.""" + +import asyncio +import json + +import httpx +import pytest +from click.testing import CliRunner + +from prescale_cli.journeys.model import ( + JourneyError, + TemplateError, + load_dir, + parse, + render, +) +from prescale_cli.journeys.replay import check +from prescale_cli.main import cli + +CHECKOUT = { + "version": 1, + "name": "checkout", + "type": "checkout", + "steps": [ + {"name": "log in", + "request": {"method": "POST", "path": "/api/login", + "json": {"email": "{{unique.email}}", "password": "{{env.PW}}"}}, + "expect": {"status": 200}, + "extract": {"token": "body.token"}}, + {"name": "add to cart", + "request": {"method": "POST", "path": "/api/cart", + "headers": {"Authorization": "Bearer {{token}}"}}, + "expect": {"status": [200, 201]}, + "extract": {"cart_id": "body.id"}}, + {"name": "place order", + "request": {"method": "POST", "path": "/api/carts/{{cart_id}}/order", + "headers": {"Authorization": "Bearer {{token}}"}}, + "think": "500ms"}, + ], +} + + +def _with(**changes): + doc = json.loads(json.dumps(CHECKOUT)) + doc.update(changes) + return doc + + +def _step(i, **changes): + doc = json.loads(json.dumps(CHECKOUT)) + doc["steps"][i].update(changes) + return doc + + +# --- the format ------------------------------------------------------------- + +def test_parse_valid_journey(): + j = parse(CHECKOUT) + assert j.name == "checkout" and j.type == "checkout" + assert j.weight == 5 # the type's default share + assert [s.label for s in j.steps] == [ + "POST /api/login", "POST /api/cart", "POST /api/carts/{{cart_id}}/order"] + assert j.steps[1].expect == [200, 201] + assert j.steps[2].expect is None and j.steps[2].think == 0.5 + assert j.writes + assert j.env_names() == {"PW"} + + +def test_read_only_journey_does_not_write(): + j = parse({"version": 1, "name": "browse", "steps": [{"request": {"path": "/"}}]}) + assert j.type == "browse" and not j.writes + assert j.steps[0].method == "GET" and j.steps[0].name == "step 1" + + +@pytest.mark.parametrize("doc, message", [ + (_with(version=2), "version must be 1"), + (_with(name="Check Out"), "name must be lowercase"), + (_with(type="spam"), "type must be one of"), + (_with(weight=0), "weight must be a positive number"), + (_with(steps=[]), "steps must be a non-empty list"), + (_with(extra=True), "unknown field(s): extra"), + (_step(0, request={"method": "POST", "path": "https://evil.example/x"}), "path must start with /"), + (_step(0, request={"method": "BREW", "path": "/"}), "method BREW isn't supported"), + (_step(0, expect={"status": "ok"}), "HTTP status codes"), + (_step(0, extract={"token": "cookie.sid"}), "body. or header."), + (_step(0, think="soon"), "think must be seconds"), +]) +def test_parse_rejects_bad_journeys(doc, message): + with pytest.raises(JourneyError, match=message.replace("(", r"\(").replace(")", r"\)")): + parse(doc) + + +@pytest.mark.parametrize("jtype", ["side_effect", "destructive"]) +def test_refused_types_are_never_loaded(jtype): + with pytest.raises(JourneyError, match="never replayed"): + parse(_with(type=jtype)) + + +def test_placeholder_must_be_extracted_by_an_earlier_step(): + doc = json.loads(json.dumps(CHECKOUT)) + del doc["steps"][0]["extract"] + with pytest.raises(JourneyError, match=r"uses \{\{token\}\}, but no earlier step"): + parse(doc) + + +def test_unknown_unique_placeholder_is_rejected(): + doc = _step(0, request={"method": "GET", "path": "/u/{{unique.phone}}"}) + with pytest.raises(JourneyError, match="unique.email or unique.id"): + parse(doc) + + +@pytest.mark.parametrize("request_part", [ + {"json": {"email": "a@b.co", "password": "hunter2"}}, + {"headers": {"Authorization": "Bearer abc123"}}, + {"json": {"user": {"api_key": "sk-live"}}}, +]) +def test_literal_secrets_are_refused(request_part): + doc = _step(0, request={"method": "POST", "path": "/api/login", **request_part}) + with pytest.raises(JourneyError, match="literal value"): + parse(doc) + + +def test_load_dir_reports_bad_files_and_duplicate_names(tmp_path): + (tmp_path / "a.yaml").write_text(json.dumps(CHECKOUT)) + (tmp_path / "b.yml").write_text(json.dumps(CHECKOUT)) # same name + (tmp_path / "c.yaml").write_text("version: 1\nname: [oops\n") # not YAML + found, errors = load_dir(tmp_path) + assert [j.name for j in found] == ["checkout", "checkout"] + assert any("not valid YAML" in e for e in errors) + assert any("already used" in e for e in errors) + + +def test_load_dir_missing_directory_is_empty(tmp_path): + assert load_dir(tmp_path / "nope") == ([], []) + + +# --- templating ------------------------------------------------------------- + +def test_render_keeps_type_of_whole_placeholder(): + variables = {"id": 7, "token": "t1"} + assert render({"cart": "{{id}}"}, variables, {}) == {"cart": 7} + assert render("/carts/{{id}}/order", variables, {}) == "/carts/7/order" + assert render("Bearer {{token}}", variables, {}) == "Bearer t1" + assert render("{{env.PW}}", {}, {"PW": "s3cret"}) == "s3cret" + + +def test_render_missing_env_is_an_error(): + with pytest.raises(TemplateError, match="PW isn't set"): + render("{{env.PW}}", {}, {}) + + +# --- replaying once ---------------------------------------------------------- + +def _shop(order_status=201, cart_path="/api/cart"): + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + if request.url.path == "/api/login": + body = json.loads(request.content) + assert body["password"] == "pw" and body["email"].endswith("@example.com") + return httpx.Response(200, json={"token": "t1"}) + assert request.headers["authorization"] == "Bearer t1" + if request.url.path == cart_path: + return httpx.Response(201, json={"id": 42}) + if request.url.path == "/api/carts/42/order": + return httpx.Response(order_status, text="pool exhausted") + return httpx.Response(404, json={"error": "not found"}) + + return httpx.MockTransport(handler), seen + + +def _check(transport): + return asyncio.run(check(parse(CHECKOUT), "http://shop.test/", env={"PW": "pw"}, + transport=transport)) + + +def test_check_replays_and_carries_extracted_values(): + transport, seen = _shop() + result = _check(transport) + assert result.ok and not result.stale + assert [s.status for s in result.steps] == [200, 201, 201] + assert str(seen[2].url) == "http://shop.test/api/carts/42/order" + + +def test_check_marks_a_changed_app_as_stale(): + transport, _ = _shop(cart_path="/api/basket") + result = _check(transport) + assert not result.ok and result.stale + assert len(result.steps) == 2 # stops at the failing step + assert result.failed.problem.startswith("returned 404, expected 200 or 201") + + +def test_check_without_expect_fails_on_4xx_and_5xx(): + transport, _ = _shop(order_status=500) + result = _check(transport) + assert result.stale + assert "expected a status below 400: pool exhausted" in result.failed.problem + + +def test_check_unreachable_app_is_not_stale(): + def boom(request): + raise httpx.ConnectError("refused") + result = _check(httpx.MockTransport(boom)) + assert not result.ok and not result.stale + assert result.failed.problem.startswith("couldn't reach the app") + + +def test_check_missing_extract_fails_the_step(): + def handler(request): + return httpx.Response(200, json={"no_token": True}) + result = _check(httpx.MockTransport(handler)) + assert "couldn't find body.token" in result.failed.problem + + +# --- the command ------------------------------------------------------------- + +def _out(res): + return " ".join(res.output.split()) # rich wraps long lines + + +def _write(tmp_path, doc=CHECKOUT, name="checkout.yaml"): + (tmp_path / name).write_text(json.dumps(doc)) + return tmp_path + + +def test_journeys_lists_files(tmp_path): + res = CliRunner().invoke(cli, ["journeys", "--dir", str(_write(tmp_path))]) + assert res.exit_code == 0, res.output + assert "checkout" in res.output and "yes" in res.output + + +def test_verify_refuses_writes_on_a_non_local_host(tmp_path): + res = CliRunner().invoke(cli, ["journeys", "verify", "https://shop.example.com", + "--dir", str(_write(tmp_path))], env={"PW": "pw"}) + assert res.exit_code == 2 + assert "--allow-writes shop.example.com" in _out(res) + + +def test_verify_refuses_when_env_is_missing(tmp_path): + res = CliRunner().invoke(cli, ["journeys", "verify", "http://localhost:1", + "--dir", str(_write(tmp_path))], env={"PW": None}) + assert res.exit_code == 2 + assert "aren't set: PW" in _out(res) + + +def test_verify_reports_invalid_files(tmp_path): + bad = _with(name="Bad Name") + res = CliRunner().invoke(cli, ["journeys", "verify", "http://localhost:1", + "--dir", str(_write(tmp_path, bad))]) + assert res.exit_code == 2 + assert "name must be lowercase" in _out(res) + + +def test_verify_unknown_journey_name(tmp_path): + res = CliRunner().invoke(cli, ["journeys", "verify", "http://localhost:1", + "--dir", str(_write(tmp_path)), "--journey", "nope"], + env={"PW": "pw"}) + assert res.exit_code == 2 + assert "no journey named nope" in _out(res) diff --git a/examples/README.md b/examples/README.md index 175dd94..bdd75ef 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,3 +17,12 @@ PreScale reports that `/checkout` breaks first, then diagnoses it as a connection-pool ceiling and suggests raising the pool / adding pgbouncer. It's the app behind the demo in the README. Tune the `Semaphore(25)` to move the breaking point. + +### Journeys + +The shop also has a small JSON API (log in, add to cart, place an order; the order +draws from the same pool) and two journeys for it in `journeys/`: + +```bash +SHOP_PASSWORD=demo prescale journeys verify http://localhost:8400 --dir examples/journeys +``` diff --git a/examples/fragile-shop.py b/examples/fragile-shop.py index 0edb2af..2a6bd2d 100644 --- a/examples/fragile-shop.py +++ b/examples/fragile-shop.py @@ -7,10 +7,18 @@ python examples/fragile-shop.py 8400 prescale investigate http://localhost:8400 --path /search --path /product --path /checkout + +It also has a small JSON API for journeys (log in, add to cart, place an order; +the order draws from the same pool), with journey files in examples/journeys/: + + SHOP_PASSWORD=demo prescale journeys verify http://localhost:8400 --dir examples/journeys """ import asyncio +import itertools +import json import random +import secrets import sys PAGE = (b"" @@ -18,7 +26,13 @@ b'' b"acme.shop") -_REASON = {200: "OK", 404: "Not Found", 500: "Internal Server Error"} +_REASON = {200: "OK", 201: "Created", 400: "Bad Request", 401: "Unauthorized", + 404: "Not Found", 500: "Internal Server Error"} + +PRODUCTS = [{"id": 1, "name": "mug"}, {"id": 2, "name": "tee"}, {"id": 3, "name": "cap"}] +_tokens: set[str] = set() +_carts: dict[int, str] = {} # cart id -> owner token +_cart_ids = itertools.count(1) def _resp(code, body=b"ok", ctype="text/plain"): @@ -36,13 +50,21 @@ async def _handle(reader, writer, pool): if not line: return parts = line.decode(errors="replace").split() + method = parts[0] if parts else "GET" path = parts[1].split("?")[0] if len(parts) >= 2 else "/" - while True: # drain request headers + headers = {} + while True: # read request headers header = await reader.readline() if header in (b"\r\n", b"\n", b""): break + name, _, value = header.decode(errors="replace").partition(":") + headers[name.strip().lower()] = value.strip() + length = int(headers.get("content-length") or 0) + body = await reader.readexactly(length) if length else b"" - if path == "/": + if path.startswith("/api/"): + out = await _api(method, path, headers, body, pool) + elif path == "/": await asyncio.sleep(0.005) out = _resp(200, PAGE, "text/html") elif path in ("/app.css", "/app.js"): @@ -75,6 +97,49 @@ async def _handle(reader, writer, pool): pass +def _json(code, data): + return _resp(code, json.dumps(data).encode(), "application/json") + + +async def _api(method, path, headers, body, pool): + """The shop's JSON API, used by the example journeys.""" + token = headers.get("authorization", "").removeprefix("Bearer ").strip() + if method == "GET" and path == "/api/products": + await asyncio.sleep(random.uniform(0.005, 0.02)) + return _json(200, PRODUCTS) + if method == "POST" and path == "/api/login": + try: + creds = json.loads(body or b"{}") + except ValueError: + return _json(400, {"error": "send JSON"}) + if not creds.get("email") or not creds.get("password"): + return _json(401, {"error": "email and password required"}) + await asyncio.sleep(0.02) # password hashing + new = secrets.token_hex(8) + _tokens.add(new) + return _json(200, {"token": new}) + if token not in _tokens: + return _json(401, {"error": "log in first"}) + if method == "POST" and path == "/api/cart": + cart = next(_cart_ids) + _carts[cart] = token + return _json(201, {"id": cart}) + if method == "POST" and path.startswith("/api/carts/") and path.endswith("/order"): + cart = path.split("/")[3] + if not cart.isdigit() or _carts.get(int(cart)) != token: + return _json(404, {"error": "no such cart"}) + try: # grab a DB connection + await asyncio.wait_for(pool.acquire(), timeout=0.001) + except asyncio.TimeoutError: + return _json(500, {"error": "could not acquire DB connection (pool exhausted)"}) + try: + await asyncio.sleep(random.uniform(0.08, 0.12)) + return _json(201, {"order": f"o-{cart}"}) + finally: + pool.release() + return _json(404, {"error": "not found"}) + + async def main(port): pool = asyncio.Semaphore(25) # 25 DB connections server = await asyncio.start_server( diff --git a/examples/journeys/browse.yaml b/examples/journeys/browse.yaml new file mode 100644 index 0000000..b10e145 --- /dev/null +++ b/examples/journeys/browse.yaml @@ -0,0 +1,12 @@ +version: 1 +name: browse +type: browse +steps: + - name: home + request: {method: GET, path: /} + - name: list products + request: {method: GET, path: /api/products} + expect: {status: 200} + extract: {product_id: body.0.id} + - name: product page + request: {method: GET, path: /product} diff --git a/examples/journeys/checkout.yaml b/examples/journeys/checkout.yaml new file mode 100644 index 0000000..6725e2b --- /dev/null +++ b/examples/journeys/checkout.yaml @@ -0,0 +1,25 @@ +version: 1 +name: checkout +type: checkout +steps: + - name: log in + request: + method: POST + path: /api/login + json: {email: "{{unique.email}}", password: "{{env.SHOP_PASSWORD}}"} + expect: {status: 200} + extract: {token: body.token} + - name: add to cart + request: + method: POST + path: /api/cart + headers: {Authorization: "Bearer {{token}}"} + expect: {status: 201} + extract: {cart_id: body.id} + - name: place order + request: + method: POST + path: "/api/carts/{{cart_id}}/order" + headers: {Authorization: "Bearer {{token}}"} + expect: {status: 201} + think: 1s From c3b3c62be8b1d30624225a4cee772f4d87cb50f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:07:32 +0000 Subject: [PATCH 02/11] fix(journeys): keep every request on the app, tighten the secrets rule --- README.md | 2 +- cli/src/prescale_cli/commands/journeys.py | 19 ++-- cli/src/prescale_cli/journeys/model.py | 43 +++++++--- cli/src/prescale_cli/journeys/replay.py | 48 +++++++++-- cli/tests/test_journeys.py | 100 +++++++++++++++++++++- examples/fragile-shop.py | 12 ++- 6 files changed, 193 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 552c8a1..7be10e9 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ prescale journeys verify http://localhost:3000 # replay each once, as one us - `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. - **Secrets never go in the file.** A literal password, token, API key or `Authorization` value is refused; use `{{env.NAME}}`. -- Paths must start with `/`: a journey only calls the app it's run against. +- Paths start with `/` and are relative to the URL you give (`http://host/app` + `/api/cart` is `http://host/app/api/cart`). A journey only calls that app: a request or redirect to another host fails the step. - Journeys that change data (POST, PUT, PATCH, DELETE) run against local hosts, or a host you name with `--allow-writes staging.myapp.com`. Types `side_effect` (emails, SMS, invites) and `destructive` are never replayed. Try it on the example shop: `python examples/fragile-shop.py 8400`, then `SHOP_PASSWORD=demo prescale journeys verify http://localhost:8400 --dir examples/journeys`. diff --git a/cli/src/prescale_cli/commands/journeys.py b/cli/src/prescale_cli/commands/journeys.py index 626b39a..23f112a 100644 --- a/cli/src/prescale_cli/commands/journeys.py +++ b/cli/src/prescale_cli/commands/journeys.py @@ -48,7 +48,7 @@ def journeys(ctx: click.Context, directory: Path) -> None: table.add_column(col) for j in found: table.add_row(j.name, j.type, f"{j.weight:g}", str(len(j.steps)), - "yes" if j.writes else "no", str(j.source)) + "yes" if j.writes else "no", escape(str(j.source))) console.print(table) @@ -94,8 +94,7 @@ def verify(url: str, directory: Path, only: tuple[str, ...], allow_writes: tuple f"aren't set: {', '.join(unset)}.") raise SystemExit(2) - base = url if url.endswith("/") else url + "/" - results = asyncio.run(_check_all(found, base, timeout=timeout, headers=headers)) + results = asyncio.run(_check_all(found, url, timeout=timeout, headers=headers)) for result in results: _print_check(result) @@ -120,7 +119,8 @@ def _load_or_exit(directory: Path) -> list[Journey]: if errors: raise SystemExit(2) if not found: - console.print(f"No journeys in {directory}. Add one as {directory}/.yaml.") + console.print(f"No journeys in {escape(str(directory))}. Add one as " + f"{escape(str(directory))}/.yaml.", highlight=False) raise SystemExit(1) return found @@ -147,17 +147,18 @@ def _print_check(result: JourneyCheck) -> None: journey = result.journey mark = "[green]✓[/green]" if result.ok else ("[yellow]stale[/yellow]" if result.stale else "[red]✗[/red]") - console.print(f"\n{mark} [bold]{journey.name}[/bold] [dim]({journey.type}, " + console.print(f"\n{mark} [bold]{escape(journey.name)}[/bold] [dim]({journey.type}, " f"{len(journey.steps)} steps)[/dim]") for i, step in enumerate(journey.steps, 1): done = result.steps[i - 1] if i <= len(result.steps) else None + name, label = escape(step.name), escape(step.label) if done is None: - console.print(f" [dim]{i}. {step.name} {step.label} (not run)[/dim]") + console.print(f" [dim]{i}. {name} {label} (not run)[/dim]", highlight=False) elif done.ok: - console.print(f" {i}. {step.name} [dim]{step.label}[/dim] " - f"{done.status} {done.latency * 1000:.0f}ms") + console.print(f" {i}. {name} [dim]{label}[/dim] " + f"{done.status} {done.latency * 1000:.0f}ms", highlight=False) else: - console.print(f" {i}. {step.name} [dim]{step.label}[/dim] " + console.print(f" {i}. {name} [dim]{label}[/dim] " f"[red]{escape(done.problem or '')}[/red]", highlight=False) if done.status is not None and done.status >= 500: console.print(" [dim]A 5xx with no load is usually a bug in the app, " diff --git a/cli/src/prescale_cli/journeys/model.py b/cli/src/prescale_cli/journeys/model.py index 1ca708b..699133f 100644 --- a/cli/src/prescale_cli/journeys/model.py +++ b/cli/src/prescale_cli/journeys/model.py @@ -24,11 +24,14 @@ from __future__ import annotations +import json +import math import re import secrets from dataclasses import dataclass, field from pathlib import Path from typing import Any +from urllib.parse import parse_qsl, urlsplit import yaml @@ -48,8 +51,14 @@ READ_METHODS = {"GET", "HEAD", "OPTIONS"} _PLACEHOLDER = re.compile(r"\{\{\s*([A-Za-z_][\w.]*)\s*\}\}") -_SENSITIVE_KEY = re.compile(r"pass(word|wd)?|secret|token|api[-_]?key|authorization|cookie", - re.IGNORECASE) +# A key is secret-looking when it is, or ends with, one of these words: +# password, client_secret, x-api-key, access_token, Authorization, Cookie. +_SENSITIVE_KEY = re.compile( + r"(^|[-_.])(password|passwd|pwd|secret|token|api[-_]?key|authorization|cookie)$", + re.IGNORECASE) +# The secret itself must be a placeholder, optionally after an auth scheme. +_SECRET_OK = re.compile(r"((Bearer|Basic|Token)\s+)?\{\{\s*[A-Za-z_][\w.]*\s*\}\}", + re.IGNORECASE) _UNIQUE = {"email", "id"} _SOURCES = ("body", "header") @@ -149,7 +158,8 @@ def parse(data: Any) -> Journey: if jtype not in TYPES: raise JourneyError(f"type must be one of {', '.join(TYPES)} (got {jtype!r})") weight = data.get("weight", TYPES[jtype]) - if isinstance(weight, bool) or not isinstance(weight, (int, float)) or weight <= 0: + if isinstance(weight, bool) or not isinstance(weight, (int, float)) \ + or not math.isfinite(weight) or weight <= 0: raise JourneyError("weight must be a positive number") raw_steps = data.get("steps") if not isinstance(raw_steps, list) or not raw_steps: @@ -168,7 +178,8 @@ def _parse_step(raw: Any, index: int) -> Step: name = raw.get("name") or f"step {index}" if not isinstance(name, str): raise JourneyError(f"{where}: name must be text") - where = f"step {index} ({name})" + if raw.get("name"): + where = f"step {index} ({name})" request = raw.get("request") if not isinstance(request, dict): @@ -178,9 +189,9 @@ def _parse_step(raw: Any, index: int) -> Step: if method not in METHODS: raise JourneyError(f"{where}: method {method} isn't supported") path = request.get("path") - if not isinstance(path, str) or not path.startswith("/"): - raise JourneyError(f"{where}: path must start with / (journeys only call the app " - "they're run against, never another site)") + if not isinstance(path, str) or not path.startswith("/") or path.startswith(("//", "/\\")): + raise JourneyError(f"{where}: path must start with a single / (it's relative to the " + "URL the journey runs against, and never reaches another site)") headers = request.get("headers") or {} if not isinstance(headers, dict) or not all(isinstance(v, str) for v in headers.values()): raise JourneyError(f"{where}: headers must map names to text") @@ -189,7 +200,9 @@ def _parse_step(raw: Any, index: int) -> Step: body = request.get("body") if body is not None and not isinstance(body, str): raise JourneyError(f"{where}: body must be text (use json for structured data)") - _check_secrets({"headers": headers, "json": request.get("json")}, where) + _check_secrets({"headers": headers, "json": request.get("json"), + "query": dict(parse_qsl(urlsplit(path).query)), + "body": _decode_body(body)}, where) return Step( name=name, method=method, path=path, @@ -247,6 +260,16 @@ def _only(data: dict, allowed: set[str], where: str) -> None: raise JourneyError(f"{where} has unknown field(s): {', '.join(map(str, unknown))}") +def _decode_body(body: str | None) -> Any: + """A string body as data, so secrets in it are checked too (JSON or a form).""" + if not body: + return None + try: + return json.loads(body) + except ValueError: + return dict(parse_qsl(body)) + + def _check_secrets(value: Any, where: str, key: str = "") -> None: """Refuse a literal value under a secret-looking key: secrets come from {{env.*}} (or an earlier step's extract), so they never end up committed in the file.""" @@ -256,8 +279,8 @@ def _check_secrets(value: Any, where: str, key: str = "") -> None: elif isinstance(value, list): for v in value: _check_secrets(v, where, key) - elif isinstance(value, str) and key and _SENSITIVE_KEY.search(key) \ - and not _PLACEHOLDER.search(value): + elif value is not None and key and _SENSITIVE_KEY.search(key) \ + and not (isinstance(value, str) and _SECRET_OK.fullmatch(value.strip())): raise JourneyError(f"{where}: {key} has a literal value. Put secrets in an " f"environment variable and write \"{{{{env.NAME}}}}\" instead") diff --git a/cli/src/prescale_cli/journeys/replay.py b/cli/src/prescale_cli/journeys/replay.py index b6e67ba..a5222e0 100644 --- a/cli/src/prescale_cli/journeys/replay.py +++ b/cli/src/prescale_cli/journeys/replay.py @@ -1,5 +1,9 @@ """Run a journey's steps against an app. +A step's path is relative to the URL the journey runs against: with +`http://host/app`, the path `/api/cart` is `http://host/app/api/cart`. Every +request, redirects included, must stay on that URL's scheme, host and port. + `check` replays a journey once, as a single user, to see whether it still works against the app as it is now. A step that fails its expectation here, with no load at all, means the journey is stale (the app changed), not that the app @@ -13,7 +17,6 @@ import time from dataclasses import dataclass, field from typing import Any -from urllib.parse import urljoin import httpx @@ -23,6 +26,26 @@ _SNIPPET = 160 +class OffOriginError(Exception): + """A request that would leave the app the journey runs against.""" + + +def step_url(base_url: str, path: str) -> str: + """The step's URL: PATH appended to BASE_URL, refused if it leaves its origin. + Appending (not urljoin) keeps a base path like /app and never lets a rendered + path such as //other-host/x change the host.""" + if not path.startswith("/") or path.startswith(("//", "/\\")): + raise OffOriginError(f"path {path!r} must start with a single /") + url = base_url.rstrip("/") + path + if _origin(httpx.URL(url)) != _origin(httpx.URL(base_url)): + raise OffOriginError(f"{url} isn't on {base_url}") + return url + + +def _origin(url: httpx.URL) -> tuple[str, str, int | None]: + return url.scheme, url.host, url.port + + @dataclass class StepCheck: step: Step @@ -59,8 +82,16 @@ async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, env = dict(os.environ) if env is None else env result = JourneyCheck(journey) variables: dict[str, Any] = fresh_uniques() + origin = _origin(httpx.URL(base_url)) + + async def same_origin(request: httpx.Request) -> None: + # Runs for redirect hops too, so an app can't bounce a write elsewhere. + if _origin(request.url) != origin: + raise OffOriginError(f"redirected to {request.url.host}, outside the app") + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True, - headers=client_headers(headers), transport=transport) as client: + headers=client_headers(headers), transport=transport, + event_hooks={"request": [same_origin]}) as client: for step in journey.steps: outcome = await run_step(client, step, base_url, variables, env) result.steps.append(outcome) @@ -75,18 +106,21 @@ async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, into VARIABLES for the steps after it.""" outcome = StepCheck(step) try: - path = render(step.path, variables, env) - req_headers = render(step.headers, variables, env) + url = step_url(base_url, str(render(step.path, variables, env))) + req_headers = {k: str(v) for k, v in render(step.headers, variables, env).items()} payload = render(step.json, variables, env) if step.json is not None else None body = render(step.body, variables, env) if step.body is not None else None - except TemplateError as exc: + except (TemplateError, OffOriginError) as exc: outcome.problem = str(exc) return outcome start = time.perf_counter() try: - resp = await client.request(step.method, urljoin(base_url, path), - headers=req_headers, json=payload, content=body) + resp = await client.request(step.method, url, headers=req_headers, json=payload, + content=body) + except OffOriginError as exc: + outcome.problem = str(exc) + return outcome except httpx.TimeoutException: outcome.problem = "timed out" return outcome diff --git a/cli/tests/test_journeys.py b/cli/tests/test_journeys.py index 68c8142..889c3e0 100644 --- a/cli/tests/test_journeys.py +++ b/cli/tests/test_journeys.py @@ -79,7 +79,10 @@ def test_read_only_journey_does_not_write(): (_with(weight=0), "weight must be a positive number"), (_with(steps=[]), "steps must be a non-empty list"), (_with(extra=True), "unknown field(s): extra"), - (_step(0, request={"method": "POST", "path": "https://evil.example/x"}), "path must start with /"), + (_step(0, request={"method": "POST", "path": "https://evil.example/x"}), "single /"), + (_step(0, request={"method": "DELETE", "path": "//evil.example/x"}), "single /"), + (_with(weight=float("nan")), "weight must be a positive number"), + (_with(weight=float("inf")), "weight must be a positive number"), (_step(0, request={"method": "BREW", "path": "/"}), "method BREW isn't supported"), (_step(0, expect={"status": "ok"}), "HTTP status codes"), (_step(0, extract={"token": "cookie.sid"}), "body. or header."), @@ -120,6 +123,30 @@ def test_literal_secrets_are_refused(request_part): parse(doc) +@pytest.mark.parametrize("request_part", [ + {"body": '{"email": "a@b.co", "password": "hunter2"}'}, + {"body": "user=a&password=hunter2"}, + {"path": "/a?api_key=sk_live_123"}, + {"json": {"password": 123456}}, + {"headers": {"Authorization": "Basic dXNlcjpwdw== {{unique.id}}"}}, +]) +def test_literal_secrets_are_refused_anywhere(request_part): + request_part.setdefault("path", "/api/login") + doc = _step(0, request={"method": "POST", "path": "/api/login", **request_part}) + with pytest.raises(JourneyError, match="literal value"): + parse(doc) + + +@pytest.mark.parametrize("request_part", [ + {"json": {"passenger_name": "Ada", "passport": "X1", "token_type": "bearer", + "cookie_consent": "yes", "bypass": "no"}}, + {"headers": {"Authorization": "Bearer {{env.TOKEN}}", "X-Api-Key": "{{env.KEY}}"}}, + {"headers": {"Authorization": "Basic {{env.BASIC}}"}}, +]) +def test_ordinary_fields_and_placeholder_secrets_are_allowed(request_part): + parse(_step(0, request={"method": "POST", "path": "/api/login", **request_part})) + + def test_load_dir_reports_bad_files_and_duplicate_names(tmp_path): (tmp_path / "a.yaml").write_text(json.dumps(CHECKOUT)) (tmp_path / "b.yml").write_text(json.dumps(CHECKOUT)) # same name @@ -213,6 +240,68 @@ def handler(request): assert "couldn't find body.token" in result.failed.problem +def _one_step(path="/next", **request): + return parse({"version": 1, "name": "j", "steps": [ + {"request": {"method": "GET", "path": "/start"}, "extract": {"next": "body.next"}}, + {"request": {"method": "POST", "path": path, **request}}]}) + + +def test_rendered_path_cannot_change_host(): + seen = [] + + def handler(request): + seen.append(str(request.url)) + if request.url.path == "/start": + return httpx.Response(200, json={"next": "/evil.example/x", "pid": 1}) + return httpx.Response(200) + + j = _one_step(path="/{{next}}") + result = asyncio.run(check(j, "http://shop.test", env={}, transport=httpx.MockTransport(handler))) + assert seen == ["http://shop.test/start"] # nothing sent off the app + assert not result.ok and "single /" in result.failed.problem + + +def test_base_url_path_prefix_is_kept(): + seen = [] + + def handler(request): + seen.append(request.url.path) + return httpx.Response(200, json={"next": "x"}) + + asyncio.run(check(_one_step(), "http://shop.test/app/", env={}, + transport=httpx.MockTransport(handler))) + assert seen == ["/app/start", "/app/next"] + + +def test_redirect_to_another_host_fails_the_step(): + seen = [] + + def handler(request): + seen.append(request.url.host) + if request.url.path == "/start": + return httpx.Response(200, json={"next": "x"}) + return httpx.Response(307, headers={"Location": "https://evil.example/steal"}) + + result = asyncio.run(check(_one_step(), "http://shop.test", env={}, + transport=httpx.MockTransport(handler))) + assert "evil.example" not in seen + assert not result.ok and "outside the app" in result.failed.problem + + +def test_extracted_number_in_a_header_is_sent_as_text(): + sent = {} + + def handler(request): + if request.url.path == "/start": + return httpx.Response(200, json={"next": 7}) + sent["x-id"] = request.headers["x-id"] + return httpx.Response(200) + + result = asyncio.run(check(_one_step(headers={"X-Id": "{{next}}"}), "http://shop.test", + env={}, transport=httpx.MockTransport(handler))) + assert result.ok and sent["x-id"] == "7" + + # --- the command ------------------------------------------------------------- def _out(res): @@ -252,6 +341,15 @@ def test_verify_reports_invalid_files(tmp_path): assert "name must be lowercase" in _out(res) +def test_verify_prints_markup_like_names_safely(tmp_path): + doc = {"version": 1, "name": "j", "steps": [ + {"name": "[/bold] oops", "request": {"method": "GET", "path": "/"}}]} + res = CliRunner().invoke(cli, ["journeys", "verify", "http://127.0.0.1:9", + "--dir", str(_write(tmp_path, doc))]) + assert res.exit_code == 1, res.output + assert "[/bold] oops" in res.output + + def test_verify_unknown_journey_name(tmp_path): res = CliRunner().invoke(cli, ["journeys", "verify", "http://localhost:1", "--dir", str(_write(tmp_path)), "--journey", "nope"], diff --git a/examples/fragile-shop.py b/examples/fragile-shop.py index 2a6bd2d..c176011 100644 --- a/examples/fragile-shop.py +++ b/examples/fragile-shop.py @@ -59,8 +59,12 @@ async def _handle(reader, writer, pool): break name, _, value = header.decode(errors="replace").partition(":") headers[name.strip().lower()] = value.strip() - length = int(headers.get("content-length") or 0) - body = await reader.readexactly(length) if length else b"" + length = headers.get("content-length") or "0" + if not length.isdigit(): + writer.write(_resp(400, b"bad Content-Length")) + await writer.drain() + return + body = await reader.readexactly(int(length)) if int(length) else b"" if path.startswith("/api/"): out = await _api(method, path, headers, body, pool) @@ -111,7 +115,9 @@ async def _api(method, path, headers, body, pool): try: creds = json.loads(body or b"{}") except ValueError: - return _json(400, {"error": "send JSON"}) + creds = None + if not isinstance(creds, dict): + return _json(400, {"error": "send a JSON object"}) if not creds.get("email") or not creds.get("password"): return _json(401, {"error": "email and password required"}) await asyncio.sleep(0.02) # password hashing From 6ac5abed647ed156528ee63097eafc2362c109e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:27:06 +0000 Subject: [PATCH 03/11] feat(journeys): prescale run replays journeys at load --- CHANGELOG.md | 6 + README.md | 17 +- cli/src/prescale_cli/commands/journeys.py | 20 ++- cli/src/prescale_cli/commands/run.py | 164 ++++++++++++++++-- cli/src/prescale_cli/journeys/load.py | 122 +++++++++++++ cli/src/prescale_cli/journeys/replay.py | 112 ++++++++++-- cli/src/prescale_cli/loadtest.py | 39 +++-- cli/src/prescale_cli/render.py | 38 ++++ .../prescale_cli/schema/result.schema.json | 63 +++++++ cli/tests/test_journeys_load.py | 152 ++++++++++++++++ 10 files changed, 675 insertions(+), 58 deletions(-) create mode 100644 cli/src/prescale_cli/journeys/load.py create mode 100644 cli/tests/test_journeys_load.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e2909bb..6204b14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ - Journeys that change data only run against local hosts or hosts you allow with `--allow-writes`; journeys that send messages or delete data are never replayed. - The example shop has a JSON API and two example journeys in `examples/journeys/`. +- **`prescale run` replays journeys at load.** When `prescale/journeys/` has journeys, virtual + users walk whole journeys (with each step's think time and their own cookies) instead of hitting + single URLs. Each journey is ramped on its own; the report names the journey and the step that + break first, with per-step timings, and saved runs carry a `journeys` field (plus + `verdict.culprit_journey`). Stale journeys are skipped with the reason. `--journey` picks + journeys; `--urls` tests URLs as before. ### Fixed - **`prescale mcp` works on a fresh install again.** `pip install 'prescale[mcp]'` picked up the new diff --git a/README.md b/README.md index 7be10e9..16371a8 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ From source: `git clone https://github.com/pyjeebz/PreScale.git && pip install . | [`run `](#run--what-breaks-first) | Ramp traffic; report what breaks first and at what load | | [`investigate `](#investigate--and-why) | …and *why* it breaks, plus how to fix it (`--app` to start your app first: [replica mode](#replica-mode--test-your-app-not-your-host)) | | [`audit `](#audit--hygiene-no-load) | Load-free HTTP hygiene check (compression, caching, CDN, HTTP/2) | -| [`journeys`](#journeys--your-users-real-paths) | List your journey files; `journeys verify ` checks they still replay | +| [`journeys`](#journeys--your-users-real-paths) | List your journey files; `journeys verify ` checks they still replay. `run` replays them at load | | [`compare`](#ci--gate-on-capacity) | Capacity diff of two runs; regression gating for CI | | [`profiles`](#frame-it-as-a-launch) | Launch scenarios for `--profile` | | [`history` / `show`](#saved-runs-and-the-json-contract) | List and re-open saved runs | @@ -198,10 +198,23 @@ prescale journeys verify http://localhost:3000 # replay each once, as one us `verify` stops at the first step that doesn't answer as expected and calls the journey **stale**: the app changed, or it's broken. It exits non-zero, so it works as a CI check. +When `prescale/journeys/` has journeys, **`prescale run` replays them at load**: each virtual user walks a whole journey (logs in, adds to cart, orders), pauses for each step's `think` time, then starts again as a new user. Each journey is ramped on its own, so its verdict doesn't depend on any traffic mix, and the report names the journey and the step that break first: + +``` + Journeys (each ramped on its own) + Journey Survives Breaks at step Why + browse 400+ (held) — + checkout ~20 (20–30) 3. place order POST /api/carts/{{cart_id}}/order + Server returned 5xx under load… +``` + +A stale journey isn't load-tested (a failure with no load says nothing about capacity); it's listed with the reason. `--journey NAME` picks journeys, and `--urls` (or `--path`, `--from-sitemap`) tests URLs instead. + - `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. - **Secrets never go in the file.** A literal password, token, API key or `Authorization` value is refused; use `{{env.NAME}}`. - Paths start with `/` and are relative to the URL you give (`http://host/app` + `/api/cart` is `http://host/app/api/cart`). A journey only calls that app: a request or redirect to another host fails the step. -- Journeys that change data (POST, PUT, PATCH, DELETE) run against local hosts, or a host you name with `--allow-writes staging.myapp.com`. Types `side_effect` (emails, SMS, invites) and `destructive` are never replayed. +- Journeys that change data (POST, PUT, PATCH, DELETE) run against local hosts, or a host you name with `--allow-writes staging.myapp.com` (on `verify` and `run`). Types `side_effect` (emails, SMS, invites) and `destructive` are never replayed. +- Each run of a journey has its own cookies and extracted values, so one user's login never leaks to another. Try it on the example shop: `python examples/fragile-shop.py 8400`, then `SHOP_PASSWORD=demo prescale journeys verify http://localhost:8400 --dir examples/journeys`. diff --git a/cli/src/prescale_cli/commands/journeys.py b/cli/src/prescale_cli/commands/journeys.py index 23f112a..5918f59 100644 --- a/cli/src/prescale_cli/commands/journeys.py +++ b/cli/src/prescale_cli/commands/journeys.py @@ -87,12 +87,8 @@ def verify(url: str, directory: Path, only: tuple[str, ...], allow_writes: tuple found = [j for j in found if j.name in only] host = (parsed.hostname or "").lower() - _refuse_unallowed_writes(found, host, {h.lower() for h in allow_writes}) - unset = sorted({name for j in found for name in j.env_names()} - set(os.environ)) - if unset: - console.print(f"[red]Error:[/red] the journeys read environment variable(s) that " - f"aren't set: {', '.join(unset)}.") - raise SystemExit(2) + refuse_unallowed_writes(found, host, {h.lower() for h in allow_writes}) + refuse_missing_env(found) results = asyncio.run(_check_all(found, url, timeout=timeout, headers=headers)) for result in results: @@ -125,7 +121,17 @@ def _load_or_exit(directory: Path) -> list[Journey]: return found -def _refuse_unallowed_writes(found: list[Journey], host: str, allowed: set[str]) -> None: +def refuse_missing_env(found: list[Journey]) -> None: + """Stop before any request if a journey reads an environment variable that + isn't set, instead of failing a step halfway through.""" + unset = sorted({name for j in found for name in j.env_names()} - set(os.environ)) + if unset: + console.print(f"[red]Error:[/red] the journeys read environment variable(s) that " + f"aren't set: {', '.join(unset)}.") + raise SystemExit(2) + + +def refuse_unallowed_writes(found: list[Journey], host: str, allowed: set[str]) -> None: """Journeys that change data run on local hosts, or on hosts named with --allow-writes. Anywhere else they could change real data.""" if host in _LOCAL_HOSTS or host in allowed: diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index f260d62..df01f9c 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -13,12 +13,17 @@ import click from rich.console import Console +from rich.markup import escape from rich.panel import Panel from prescale_cli.cloud import CloudError, require_credentials from prescale_cli.commands.hostcheck import confirm_host, host_permission_option +from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica from prescale_cli.commands.upload import header_option, upload_after_run, upload_option +from prescale_cli.journeys.load import JourneyRun, first_to_break, journeys_block, ramp +from prescale_cli.journeys.model import Journey, load_dir +from prescale_cli.journeys.replay import check from prescale_cli.live import LiveRamp, ramp_progress from prescale_cli.loadtest import ( LoadError, @@ -82,6 +87,14 @@ help="Exit non-zero if it survives fewer than N users (a CI gate).") @click.option("--profile", "profile_name", default=None, help="Frame the run as a launch scenario (see `prescale profiles`).") +@click.option("--journey", "only", multiple=True, + help="Replay only this journey (repeatable). See `prescale journeys`.") +@click.option("--urls", "urls_only", is_flag=True, + help="Test URLs, not journeys, even when prescale/journeys/ has some.") +@click.option("--allow-writes", "allow_writes", multiple=True, metavar="HOST", + help="Let journeys that change data run against this non-local host " + "(repeatable).") +@dir_option @app_option @app_timeout_option @host_permission_option @@ -93,10 +106,16 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, repeat: int, think_time: float, ignore_robots: bool, yes: bool, as_json: bool, html_path: str | None, store: str | None, no_save: bool, fail_under: int | None, - profile_name: str | None, app: str | None, app_timeout: float, - host_permission: bool, headers: dict[str, str], upload: bool) -> None: + profile_name: str | None, only: tuple[str, ...], urls_only: bool, + allow_writes: tuple[str, ...], directory: Path, app: str | None, + app_timeout: float, host_permission: bool, headers: dict[str, str], + upload: bool) -> None: """Load test URL and report what breaks first. + When prescale/journeys/ has journeys, each one is replayed at load (users + walk the whole journey: log in, add to cart, order) and the report names the + journey and step that break first. Use --urls to test URLs instead. + \b Examples: prescale run http://localhost:8000 @@ -128,10 +147,16 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, max_users = prof.peak_users think_time = prof.think_time_s + journeys = _journeys_to_run(directory, only, urls_only=urls_only, + url_mode=bool(paths or from_sitemap or method.upper() != "GET")) + host = (parsed.hostname or "").lower() + if journeys: + refuse_unallowed_writes(journeys, host, {h.lower() for h in allow_writes}) + refuse_missing_env(journeys) + if app: start_replica(app, url, timeout=app_timeout, console=console, quiet=as_json) - host = (parsed.hostname or "").lower() is_local = host in _LOCAL_HOSTS if not is_local and not yes: if as_json: @@ -151,6 +176,30 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, confirm_host(url, console=console, permitted=host_permission, as_json=as_json, timeout=timeout, headers=headers) + config = { + "method": method, + "max_users": max_users, + "stage_seconds": stage_seconds, + "latency_wall_s": latency_wall, + "error_threshold": error_threshold, + "max_rps": max_rps, + "warmup": warmup, + "repeat": repeat, + "think_time_s": think_time, + "fail_under": fail_under, + "profile": prof.name if prof else None, + } + finish = dict(prof=prof, no_save=no_save, store=store, html_path=html_path, + as_json=as_json, upload=upload, fail_under=fail_under) + + if journeys: + report, result, live_mode = _run_journeys( + journeys, url, config, latency_wall=latency_wall, + error_threshold=error_threshold, timeout=timeout, headers=headers, + replica=bool(app), as_json=as_json) + _finish(result, report, live_mode=live_mode, **finish) + return + extra: list[str] = [] if from_sitemap: if not as_json: @@ -203,22 +252,15 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, report = analyze(stages, latency_wall=latency_wall, error_threshold=error_threshold, rate_capped=max_rps is not None) - - config = { - "method": method, - "max_users": max_users, - "stage_seconds": stage_seconds, - "latency_wall_s": latency_wall, - "error_threshold": error_threshold, - "max_rps": max_rps, - "warmup": warmup, - "repeat": repeat, - "think_time_s": think_time, - "fail_under": fail_under, - "profile": prof.name if prof else None, - } result = build_result(report, url=url, targets=targets, config=config, warning=warning, replica=bool(app)) + _finish(result, report, live_mode=live_mode, **finish) + + +def _finish(result: dict, report, *, prof, no_save: bool, store: str | None, + html_path: str | None, as_json: bool, upload: bool, fail_under: int | None, + live_mode: bool) -> None: + """Save, render, upload and gate a finished run (URLs or journeys alike).""" if prof is not None: result["profile"] = scenario_block(prof, report.survives_users) saved_path = None if no_save else write_result(result, store=store) @@ -244,3 +286,91 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, Console(stderr=True).print( f"[red]✗ fail-under:[/red] survives ~{report.survives_users} < {fail_under}") raise SystemExit(1) + + +def _journeys_to_run(directory: Path, only: tuple[str, ...], *, urls_only: bool, + url_mode: bool) -> list[Journey]: + """The journeys this run replays: none in URL mode (--urls, --path, + --from-sitemap or a non-GET --method), else every valid file in DIRECTORY, + narrowed by --journey. A broken journey file stops the run rather than + silently falling back to testing URLs.""" + if urls_only or url_mode: + if only: + console.print("[red]Error:[/red] --journey can't be combined with --urls, " + "--path, --from-sitemap or --method.") + raise SystemExit(2) + return [] + found, errors = load_dir(directory) + for error in errors: + console.print(f"[red]Error:[/red] {escape(error)}", highlight=False) + if errors: + raise SystemExit(2) + if only: + missing = sorted(set(only) - {j.name for j in found}) + if missing: + console.print(f"[red]Error:[/red] no journey named {', '.join(missing)} " + f"in {escape(str(directory))}.", highlight=False) + raise SystemExit(2) + found = [j for j in found if j.name in only] + return found + + +def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wall: float, + error_threshold: float, timeout: float, headers: dict[str, str], + replica: bool, as_json: bool) -> tuple: + """The isolation pass: check each journey once, then ramp each one that + replays on its own. Returns (report, result, live_mode) for the first + journey to break, with every journey's verdict in result["journeys"].""" + if not as_json: + console.print(f"\n[bold]PreScale[/bold] — replaying {len(journeys)} " + f"journey{'s' if len(journeys) != 1 else ''} against [cyan]{url}[/cyan]") + checks = asyncio.run(_check_all(journeys, url, timeout=timeout, headers=headers)) + runs = [JourneyRun(j, c) for j, c in zip(journeys, checks)] + levels = default_levels(config["max_users"]) + live_mode = False + for run in runs: + name = escape(run.journey.name) + if not run.check.ok: + failed = run.check.failed + what = "stale" if run.check.stale else "couldn't run" + console.print(f"[yellow]⚠ {name}: {what}, not load-tested[/yellow] " + f"[dim]({escape(failed.step.name)}: {escape(failed.problem or '')})" + "[/dim]", highlight=False) + continue + if not as_json: + console.print(f"\n[bold]Journey {name}[/bold] [dim]({run.journey.type}, " + f"{len(run.journey.steps)} steps)[/dim]", highlight=False) + progress = ramp_progress(console, latency_wall=latency_wall, as_json=as_json) + live_mode = isinstance(progress, LiveRamp) + try: + with progress: + stages, run.warning = asyncio.run(ramp( + run.journey, url, levels=levels, stage_seconds=config["stage_seconds"], + timeout=timeout, max_rps=config["max_rps"], warmup=config["warmup"], + repeat=config["repeat"], headers=headers, + progress_cb=progress.starting, on_stage=progress.finished)) + except LoadError as exc: + console.print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) + run.report = analyze(stages, latency_wall=latency_wall, + error_threshold=error_threshold, + rate_capped=config["max_rps"] is not None) + + worst = first_to_break(runs) + if worst is None: + console.print("[red]Error:[/red] no journey replays against this app, so there's " + "nothing to load-test. Run `prescale journeys verify` for details.") + raise SystemExit(1) + warnings = list(dict.fromkeys(r.warning for r in runs if r.warning)) + result = build_result(worst.report, url=url, + targets=[s.label for s in worst.journey.steps], + config={**config, "mode": "journeys"}, + warning=" ".join(warnings) or None, replica=replica) + result["verdict"]["culprit_journey"] = worst.journey.name + result["journeys"] = journeys_block(runs) + return worst.report, result, live_mode + + +async def _check_all(journeys: list[Journey], url: str, *, timeout: float, + headers: dict[str, str]): + return [await check(j, url, timeout=timeout, headers=headers) for j in journeys] diff --git a/cli/src/prescale_cli/journeys/load.py b/cli/src/prescale_cli/journeys/load.py new file mode 100644 index 0000000..e398ecf --- /dev/null +++ b/cli/src/prescale_cli/journeys/load.py @@ -0,0 +1,122 @@ +"""Load-test journeys: the isolation pass. + +Each journey is first replayed once (`check`). A journey that fails there is +stale or unreachable and isn't ramped: a failure with no load says nothing about +capacity. Every journey that replays is then ramped on its own, so its verdict +(how many users it survives, the step that breaks first, why) doesn't depend on +any traffic mix. The run's overall verdict is the first journey to break. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import httpx + +from prescale_cli.journeys.model import Journey +from prescale_cli.journeys.replay import JourneyCheck, client_options, journey_vu +from prescale_cli.loadtest import RunReport, StageResult, run_loadtest + + +@dataclass +class JourneyRun: + journey: Journey + check: JourneyCheck + report: RunReport | None = None + warning: str | None = None + + @property + def status(self) -> str: + if self.report is not None: + return "ok" + return "stale" if self.check.stale else "unreachable" + + +async def ramp(journey: Journey, base_url: str, *, levels: list[int], stage_seconds: float, + timeout: float = 10.0, max_rps: float | None = None, warmup: bool = True, + repeat: int = 1, headers: dict[str, str] | None = None, + env: dict[str, str] | None = None, progress_cb=None, on_stage=None, + transport: httpx.AsyncBaseTransport | None = None, + ) -> tuple[list[StageResult], str | None]: + """Ramp virtual users who each walk JOURNEY, through LEVELS.""" + return await run_loadtest( + [base_url], levels=levels, stage_seconds=stage_seconds, timeout=timeout, + max_rps=max_rps, warmup=warmup, repeat=repeat, headers=headers, + host_permission=True, # the caller has already checked the host's rules + progress_cb=progress_cb, on_stage=on_stage, transport=transport, + vu=journey_vu(journey, base_url, env), client_options=client_options(base_url), + ) + + +def first_to_break(runs: list[JourneyRun]) -> JourneyRun | None: + """The ramped journey with the lowest capacity (ties: the one that broke).""" + ramped = [r for r in runs if r.report is not None] + if not ramped: + return None + return min(ramped, key=lambda r: (r.report.survives_users, + r.report.onset_users is None)) + + +def journeys_block(runs: list[JourneyRun]) -> list[dict]: + """The Result's `journeys` field: one entry per journey, ramped or not.""" + return [_entry(run) for run in runs] + + +def _entry(run: JourneyRun) -> dict: + journey = run.journey + entry: dict = {"name": journey.name, "type": journey.type, "weight": journey.weight, + "status": run.status, "reason": None} + if run.report is None: + failed = run.check.failed + index = run.check.steps.index(failed) + 1 if failed else None + entry["reason"] = f"step {index} ({failed.step.name}): {failed.problem}" if failed \ + else None + return entry + + report = run.report + stage = _stage_at(report) + culprit = None + if report.culprit_route: + index = next((i for i, s in enumerate(journey.steps, 1) + if s.label == report.culprit_route), None) + if index is not None: + culprit = {"index": index, "name": journey.steps[index - 1].name, + "route": report.culprit_route} + entry.update({ + "survives_users": report.survives_users, + "max_tested": report.max_tested, + "onset_users": report.onset_users, + "onset_reason": report.onset_reason, + "confidence": {"survives_low": report.survives_low, + "survives_high": report.survives_high, "stable": report.stable}, + "culprit_step": culprit, + "bottleneck": report.bottleneck, + "steps_at_users": stage.users if stage else None, + "steps": [_step_stats(i, s, stage) for i, s in enumerate(journey.steps, 1)], + }) + return entry + + +def _stage_at(report: RunReport) -> StageResult | None: + """The stage the step timings come from: where it broke, else the last one.""" + if not report.stages: + return None + if report.onset_users is not None: + for stage in report.stages: + if stage.users == report.onset_users: + return stage + return report.stages[-1] + + +def _step_stats(index: int, step, stage: StageResult | None) -> dict: + stat = stage.routes.get(step.label) if stage else None + if stat is None: + return {"index": index, "name": step.name, "route": step.label, "requests": 0, + "p50_ms": None, "p95_ms": None, "error_rate": None} + has_latency = bool(stat.latencies) + return { + "index": index, "name": step.name, "route": step.label, "requests": stat.total, + "p50_ms": round(stat.pct(0.50) * 1000) if has_latency else None, + "p95_ms": round(stat.pct(0.95) * 1000) if has_latency else None, + "error_rate": round(stat.error_rate, 4), + } diff --git a/cli/src/prescale_cli/journeys/replay.py b/cli/src/prescale_cli/journeys/replay.py index a5222e0..69e35a3 100644 --- a/cli/src/prescale_cli/journeys/replay.py +++ b/cli/src/prescale_cli/journeys/replay.py @@ -7,15 +7,20 @@ `check` replays a journey once, as a single user, to see whether it still works against the app as it is now. A step that fails its expectation here, with no load at all, means the journey is stale (the app changed), not that the app -broke under load. Each run gets its own client, so cookies and extracted values -never leak between users. +broke under load. `journey_vu` is the same journey as a virtual user for a load +test: it walks the steps over and over until the stage ends. + +Every run of a journey has its own cookies and extracted values, so a login or +a cart never leaks from one user to another, even on a shared client. """ from __future__ import annotations +import asyncio import os import time from dataclasses import dataclass, field +from http.cookiejar import CookieJar, DefaultCookiePolicy from typing import Any import httpx @@ -46,6 +51,22 @@ def _origin(url: httpx.URL) -> tuple[str, str, int | None]: return url.scheme, url.host, url.port +def client_options(base_url: str) -> dict: + """Settings for any client that runs journeys: requests (redirect hops too) + must stay on BASE_URL's origin, and the client itself keeps no cookies, since + each run of a journey carries its own.""" + origin = _origin(httpx.URL(base_url)) + + async def same_origin(request: httpx.Request) -> None: + if _origin(request.url) != origin: + raise OffOriginError(f"redirected to {request.url.host}, outside the app") + + return { + "event_hooks": {"request": [same_origin]}, + "cookies": CookieJar(policy=DefaultCookiePolicy(allowed_domains=[])), + } + + @dataclass class StepCheck: step: Step @@ -53,6 +74,8 @@ class StepCheck: latency: float = 0.0 ok: bool = False problem: str | None = None # why it failed, in plain words + kind: str | None = None # a short failure kind for load reports + headers: httpx.Headers | None = None @dataclass @@ -82,28 +105,68 @@ async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, env = dict(os.environ) if env is None else env result = JourneyCheck(journey) variables: dict[str, Any] = fresh_uniques() - origin = _origin(httpx.URL(base_url)) - - async def same_origin(request: httpx.Request) -> None: - # Runs for redirect hops too, so an app can't bounce a write elsewhere. - if _origin(request.url) != origin: - raise OffOriginError(f"redirected to {request.url.host}, outside the app") - + jar = httpx.Cookies() async with httpx.AsyncClient(timeout=timeout, follow_redirects=True, headers=client_headers(headers), transport=transport, - event_hooks={"request": [same_origin]}) as client: + **client_options(base_url)) as client: for step in journey.steps: - outcome = await run_step(client, step, base_url, variables, env) + outcome = await run_step(client, step, base_url, variables, env, jar) result.steps.append(outcome) if not outcome.ok: break return result +def journey_vu(journey: Journey, base_url: str, env: dict[str, str] | None = None): + """JOURNEY as a load-test virtual user: walk its steps, pausing for each + step's think time, and start again with a fresh identity until the stage + deadline. A failed step ends that walk (after its pause), like a real user + giving up. Each step is recorded under its label, so the report can name the + step that broke.""" + env = dict(os.environ) if env is None else env + + async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> None: + loop = asyncio.get_running_loop() + while loop.time() < deadline: + variables: dict[str, Any] = fresh_uniques() + jar = httpx.Cookies() + for step in journey.steps: + if gate is not None and not await gate.wait(deadline): + return + if loop.time() >= deadline: + return + outcome = await run_step(client, step, base_url, variables, env, jar) + _record(sink, outcome) + # A user pauses after a step either way; restarting the instant a step + # fails would hammer the app harder than real users who hit an error. + if step.think: + remaining = deadline - loop.time() + if remaining <= 0: + return + await asyncio.sleep(min(step.think, remaining)) + if not outcome.ok: + break + + return vu + + +def _record(sink, outcome: StepCheck) -> None: + """One step's outcome into the stage's numbers. An answer the journey didn't + expect counts as a failure, even a 4xx: under load that's a step going wrong.""" + label = outcome.step.label + if outcome.status is None: + sink.record_error(label, outcome.kind or "network") + elif outcome.ok or outcome.status >= 500 or outcome.status == 429: + sink.record_response(label, outcome.status, outcome.latency, outcome.headers) + else: + sink.record_error(label, outcome.kind or f"unexpected {outcome.status}") + + async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, - variables: dict[str, Any], env: dict[str, str]) -> StepCheck: - """Send one step, judge it against `expect`, and pull its `extract` values - into VARIABLES for the steps after it.""" + variables: dict[str, Any], env: dict[str, str], + jar: httpx.Cookies) -> StepCheck: + """Send one step with this journey run's cookies (JAR), judge it against + `expect`, and pull its `extract` values into VARIABLES for the steps after it.""" outcome = StepCheck(step) try: url = step_url(base_url, str(render(step.path, variables, env))) @@ -111,24 +174,34 @@ async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, payload = render(step.json, variables, env) if step.json is not None else None body = render(step.body, variables, env) if step.body is not None else None except (TemplateError, OffOriginError) as exc: - outcome.problem = str(exc) + outcome.problem, outcome.kind = str(exc), "journey error" return outcome + request = client.build_request(step.method, url, headers=req_headers, json=payload, + content=body) + jar.set_cookie_header(request) start = time.perf_counter() try: - resp = await client.request(step.method, url, headers=req_headers, json=payload, - content=body) + resp = await client.send(request) except OffOriginError as exc: - outcome.problem = str(exc) + outcome.problem, outcome.kind = str(exc), "left the app" return outcome except httpx.TimeoutException: - outcome.problem = "timed out" + outcome.problem, outcome.kind = "timed out", "timeout" + return outcome + except httpx.ConnectError: + outcome.problem, outcome.kind = "couldn't reach the app (ConnectError)", \ + "connection refused" return outcome except httpx.HTTPError as exc: outcome.problem = f"couldn't reach the app ({exc.__class__.__name__})" + outcome.kind = "network" return outcome outcome.latency = time.perf_counter() - start outcome.status = resp.status_code + outcome.headers = resp.headers + for hop in (*resp.history, resp): + jar.extract_cookies(hop) if not step.accepts(resp.status_code): wanted = (" or ".join(map(str, step.expect)) if step.expect else "a status below 400") @@ -142,6 +215,7 @@ async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, value = extract(resp, source) if value is None: outcome.problem = f"couldn't find {source} in the response to extract {name}" + outcome.kind = "missing extract" return outcome variables[name] = value outcome.ok = True diff --git a/cli/src/prescale_cli/loadtest.py b/cli/src/prescale_cli/loadtest.py index f45ed1d..8005efb 100644 --- a/cli/src/prescale_cli/loadtest.py +++ b/cli/src/prescale_cli/loadtest.py @@ -284,13 +284,15 @@ class SaturationInfo: class _Sink: """Per-stage accumulator, keyed by route label. A 5xx/429 is a failure; - everything else with a status code counts toward latency.""" + everything else with a status code counts toward latency. Targets are URLs, + labelled by path; journeys pass their step labels as they are (`labels=False`).""" - def __init__(self) -> None: + def __init__(self, labels: bool = True) -> None: self.routes: dict[str, RouteStat] = {} + self._labels = labels def _stat(self, target: str) -> RouteStat: - label = route_label(target) + label = route_label(target) if self._labels else target stat = self.routes.get(label) if stat is None: stat = RouteStat() @@ -389,8 +391,11 @@ async def _worker(client: httpx.AsyncClient, method: str, deadline: float, async def _run_stage(client: httpx.AsyncClient, targets: list[str], method: str, users: int, duration: float, gate: _RateGate | None = None, - think_time: float = 0.0) -> StageResult: - sink = _Sink() + think_time: float = 0.0, vu=None) -> StageResult: + """Hold USERS virtual users for DURATION. By default each fires requests at + TARGETS; VU, when given, is the virtual user itself: an async callable + (client, deadline, sink, gate) that records its own requests (journeys).""" + sink = _Sink(labels=vu is None) cycle = itertools.cycle(targets) # round-robin spreads load evenly across routes loop = asyncio.get_running_loop() start = loop.time() @@ -399,10 +404,13 @@ async def _run_stage(client: httpx.AsyncClient, targets: list[str], method: str, lags: list[float] = [] watcher = asyncio.create_task(_watch_loop_lag(stop, lags)) try: - await asyncio.gather( - *(_worker(client, method, deadline, sink, lambda: next(cycle), gate, think_time) - for _ in range(users)) - ) + if vu is not None: + await asyncio.gather(*(vu(client, deadline, sink, gate) for _ in range(users))) + else: + await asyncio.gather( + *(_worker(client, method, deadline, sink, lambda: next(cycle), gate, think_time) + for _ in range(users)) + ) finally: stop.set() await watcher @@ -485,13 +493,18 @@ async def run_loadtest( transport: httpx.AsyncBaseTransport | None = None, headers: dict[str, str] | None = None, host_permission: bool = False, + vu=None, + client_options: dict | None = None, ) -> tuple[list[StageResult], str | None]: """Preflight the target, then ramp through `levels`, spreading each stage's load across `targets`. Stops early once a stage is more than `hard_stop_rate` failed. Returns (stages, warning). A host that restricts load testing (Vercel, Netlify, Cloudflare) stops the run - after the preflight unless `host_permission` says the user has permission.""" + after the preflight unless `host_permission` says the user has permission. + + `vu` replaces the default virtual user (see `_run_stage`), and + `client_options` adds settings to the HTTP client (journeys use both).""" if not targets: raise LoadError("No targets to test.") max_conns = max(levels) + 50 @@ -501,7 +514,7 @@ async def run_loadtest( gate = _RateGate(max_rps) if max_rps else None async with httpx.AsyncClient( timeout=timeout, limits=limits, follow_redirects=True, - headers=client_headers(headers), transport=transport, + headers=client_headers(headers), transport=transport, **(client_options or {}), ) as client: try: preflight = await client.request(method, targets[0]) @@ -529,7 +542,7 @@ async def run_loadtest( # level isn't cold. Still rate-gated, since it's real traffic. warmup_users, warmup_seconds = _warmup_plan(levels, stage_seconds) await _run_stage(client, targets, method, warmup_users, warmup_seconds, - gate, think_time) + gate, think_time, vu) # --repeat pools several ramps per level, so run-to-run variance (cache # state, GC, noisy neighbours) widens the confidence band honestly. @@ -539,7 +552,7 @@ async def run_loadtest( if progress_cb: progress_cb(users) stage = await _run_stage(client, targets, method, users, stage_seconds, - gate, think_time) + gate, think_time, vu) by_level.setdefault(users, []).append(stage) if on_stage: on_stage(stage) diff --git a/cli/src/prescale_cli/render.py b/cli/src/prescale_cli/render.py index 49ae8a8..b7dafb3 100644 --- a/cli/src/prescale_cli/render.py +++ b/cli/src/prescale_cli/render.py @@ -4,6 +4,7 @@ from __future__ import annotations from rich.console import Console +from rich.markup import escape from rich.panel import Panel from rich.table import Table @@ -93,6 +94,8 @@ def render_terminal(result: dict, *, show_ramp: bool = True) -> None: lines.append(f"Traffic {traffic} [dim]({VISITOR_NOTE})[/dim]") if onset_users is not None: culprit = f"{verdict['culprit_route']} " if (multi and verdict["culprit_route"]) else "" + if verdict.get("culprit_journey"): + culprit = f"{escape(verdict['culprit_journey'])} › {escape(culprit)}" if verdict["onset_reason"] == "latency": lines.append(f"Latency wall {culprit}p95 crosses " f"{latency_wall:g}s at ~{onset_users} {_u(onset_users)}.") @@ -130,6 +133,41 @@ def render_terminal(result: dict, *, show_ramp: bool = True) -> None: if multi and stages: _render_routes(result) + if result.get("journeys"): + render_journeys(result) + + +def render_journeys(result: dict) -> None: + """One row per journey: how many users it survives and the step that breaks + first. Each was ramped on its own, so no row depends on the others.""" + table = Table(show_header=True, header_style="bold magenta", + title="Journeys (each ramped on its own)") + for col, justify in (("Journey", "left"), ("Survives", "right"), + ("Breaks at step", "left"), ("Why", "left")): + table.add_column(col, justify=justify) + first = result["verdict"].get("culprit_journey") + for j in result["journeys"]: + name = escape(j["name"]) + if j["status"] != "ok": + label = "stale" if j["status"] == "stale" else "couldn't run" + table.add_row(name, f"[yellow]{label}[/yellow]", "—", + f"[dim]{escape(j.get('reason') or '')}[/dim]") + continue + conf = j.get("confidence") or {} + band = "" + if conf.get("survives_low") is not None and conf.get("survives_high") is not None \ + and conf["survives_low"] != conf["survives_high"]: + band = f" ({conf['survives_low']}–{conf['survives_high']})" + if j["onset_users"] is None: + survives = f"{j['max_tested']}+ [dim](held)[/dim]" + else: + survives = f"~{j['survives_users']}{band}" + step = j.get("culprit_step") + where = f"{step['index']}. {escape(step['name'])} [dim]{escape(step['route'])}[/dim]" \ + if step else "—" + style = "bold red" if j["name"] == first and j["onset_users"] is not None else None + table.add_row(name, survives, where, escape(j.get("bottleneck") or ""), style=style) + console.print(table) def _render_routes(result: dict) -> None: diff --git a/cli/src/prescale_cli/schema/result.schema.json b/cli/src/prescale_cli/schema/result.schema.json index bbc89b4..e2e406e 100644 --- a/cli/src/prescale_cli/schema/result.schema.json +++ b/cli/src/prescale_cli/schema/result.schema.json @@ -45,6 +45,11 @@ "replica": { "type": "boolean", "description": "True when PreScale started the app's own build locally (--app) and tested that, not a deployed host." + }, + "mode": { + "type": "string", + "enum": ["journeys"], + "description": "\"journeys\" when the run replayed journeys (prescale/journeys/); absent for a URL run." } } }, @@ -61,6 +66,10 @@ "onset_users": { "type": ["integer", "null"] }, "onset_reason": { "type": ["string", "null"], "enum": ["errors", "latency", null] }, "culprit_route": { "type": ["string", "null"] }, + "culprit_journey": { + "type": ["string", "null"], + "description": "Journey runs: the journey that breaks first. The verdict, stages and culprit_route (a step's method and path) are that journey's." + }, "bottleneck": { "type": ["string", "null"] }, "saturated": { "type": "boolean" }, "saturation_users": { "type": ["integer", "null"] }, @@ -84,6 +93,11 @@ "git_branch": { "type": ["string", "null"] } } }, + "journeys": { + "type": "array", + "description": "Present on journey runs: one entry per journey. Each journey that replays is ramped on its own; a stale or unreachable one isn't load-tested and carries the reason.", + "items": { "$ref": "#/$defs/journey" } + }, "investigation": { "type": "object", "description": "Present when `prescale investigate` diagnosed a failure.", @@ -123,6 +137,55 @@ } }, "$defs": { + "journey": { + "type": "object", + "required": ["name", "type", "weight", "status", "reason"], + "properties": { + "name": { "type": "string" }, + "type": { "type": "string" }, + "weight": { "type": "number" }, + "status": { "enum": ["ok", "stale", "unreachable"] }, + "reason": { "type": ["string", "null"], "description": "Why a stale or unreachable journey wasn't load-tested." }, + "survives_users": { "type": "integer" }, + "max_tested": { "type": "integer" }, + "onset_users": { "type": ["integer", "null"] }, + "onset_reason": { "type": ["string", "null"], "enum": ["errors", "latency", null] }, + "confidence": { + "type": "object", + "properties": { + "survives_low": { "type": ["integer", "null"] }, + "survives_high": { "type": ["integer", "null"] }, + "stable": { "type": "boolean" } + } + }, + "culprit_step": { + "type": ["object", "null"], + "properties": { + "index": { "type": "integer", "description": "1-based step number." }, + "name": { "type": "string" }, + "route": { "type": "string" } + } + }, + "bottleneck": { "type": ["string", "null"] }, + "steps_at_users": { "type": ["integer", "null"], "description": "The level the step timings come from: where it broke, else the highest tested." }, + "steps": { + "type": "array", + "items": { + "type": "object", + "required": ["index", "name", "route", "requests"], + "properties": { + "index": { "type": "integer" }, + "name": { "type": "string" }, + "route": { "type": "string" }, + "requests": { "type": "integer" }, + "p50_ms": { "type": ["integer", "null"] }, + "p95_ms": { "type": ["integer", "null"] }, + "error_rate": { "type": ["number", "null"] } + } + } + } + } + }, "stage": { "type": "object", "required": [ diff --git a/cli/tests/test_journeys_load.py b/cli/tests/test_journeys_load.py new file mode 100644 index 0000000..2c7897f --- /dev/null +++ b/cli/tests/test_journeys_load.py @@ -0,0 +1,152 @@ +"""Journeys at load: virtual users walk whole journeys, each ramped on its own.""" + +import asyncio +import json + +import httpx +from click.testing import CliRunner + +from prescale_cli.journeys.load import JourneyRun, first_to_break, journeys_block, ramp +from prescale_cli.journeys.model import parse +from prescale_cli.journeys.replay import check +from prescale_cli.loadtest import analyze +from prescale_cli.main import cli + +SHOP = { + "version": 1, "name": "checkout", "type": "checkout", + "steps": [ + {"name": "log in", "request": {"method": "POST", "path": "/api/login"}, + "expect": {"status": 200}, "extract": {"token": "body.token"}}, + {"name": "place order", + "request": {"method": "POST", "path": "/api/orders", + "headers": {"Authorization": "Bearer {{token}}"}}, + "expect": {"status": 201}}, + ], +} +BROWSE = {"version": 1, "name": "browse", "steps": [{"request": {"path": "/"}}]} + + +def _shop(pool: int): + """A mock shop whose order step fails once more than POOL orders are in + flight at once, and which checks every order carries its own login's token + and cookie.""" + state = {"in_flight": 0, "logins": 0, "mixed": 0} + + async def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/api/login": + state["logins"] += 1 + token = f"t{state['logins']}" + return httpx.Response(200, json={"token": token}, + headers={"set-cookie": f"sid={token}; Path=/"}) + if request.url.path == "/api/orders": + token = request.headers["authorization"].removeprefix("Bearer ") + if request.headers.get("cookie") != f"sid={token}": + state["mixed"] += 1 + if state["in_flight"] >= pool: + return httpx.Response(500, text="pool exhausted") + state["in_flight"] += 1 + try: + await asyncio.sleep(0.02) + finally: + state["in_flight"] -= 1 + return httpx.Response(201) + return httpx.Response(200) + + return httpx.MockTransport(handler), state + + +def _ramp(journey, transport, levels=(1, 2, 5, 10, 20)): + return asyncio.run(ramp(journey, "http://shop.test", levels=list(levels), + stage_seconds=0.2, warmup=False, transport=transport)) + + +def test_ramp_records_each_step_and_finds_the_step_that_breaks(): + transport, state = _shop(pool=4) + stages, _ = _ramp(parse(SHOP), transport) + assert set(stages[0].routes) == {"POST /api/login", "POST /api/orders"} + report = analyze(stages, latency_wall=2.0, error_threshold=0.02) + assert report.onset_reason == "errors" + assert report.culprit_route == "POST /api/orders" + assert 1 <= report.survives_users < 20 + # every order went out with its own user's token and cookie + assert state["mixed"] == 0 and state["logins"] > 20 + + +def test_journeys_block_names_the_step_and_skips_stale_journeys(): + transport, _ = _shop(pool=4) + checkout, browse = parse(SHOP), parse(BROWSE) + stages, _ = _ramp(checkout, transport) + ok = JourneyRun(checkout, asyncio.run(check(checkout, "http://shop.test", env={}, + transport=transport)), + report=analyze(stages, latency_wall=2.0, error_threshold=0.02)) + + def gone(request): + return httpx.Response(404, text="moved") + stale = JourneyRun(browse, asyncio.run(check(browse, "http://shop.test", env={}, + transport=httpx.MockTransport(gone)))) + + block = journeys_block([ok, stale]) + assert block[0]["status"] == "ok" + assert block[0]["culprit_step"] == {"index": 2, "name": "place order", + "route": "POST /api/orders"} + assert [s["route"] for s in block[0]["steps"]] == ["POST /api/login", "POST /api/orders"] + assert block[0]["steps"][1]["error_rate"] > 0 + assert block[1] == {"name": "browse", "type": "browse", "weight": 60.0, + "status": "stale", + "reason": "step 1 (step 1): returned 404, expected a status " + "below 400: moved"} + assert first_to_break([ok, stale]) is ok + json.dumps(block) # it goes into the saved Result + + +def test_first_to_break_prefers_the_lowest_capacity(): + transport, _ = _shop(pool=4) + weak = JourneyRun(parse(SHOP), None, + report=analyze(_ramp(parse(SHOP), transport)[0], + latency_wall=2.0, error_threshold=0.02)) + strong = JourneyRun(parse(BROWSE), None, + report=analyze(_ramp(parse(BROWSE), transport)[0], + latency_wall=2.0, error_threshold=0.02)) + assert strong.report.onset_users is None + assert first_to_break([strong, weak]) is weak + assert first_to_break([JourneyRun(parse(BROWSE), None)]) is None + + +# --- `prescale run` picks journeys --------------------------------------------- + +def _dir(tmp_path, *docs): + for doc in docs: + (tmp_path / f"{doc['name']}.yaml").write_text(json.dumps(doc)) + return str(tmp_path) + + +def _out(res): + return " ".join(res.output.split()) + + +def test_run_refuses_journey_writes_on_a_non_local_host(tmp_path): + res = CliRunner().invoke(cli, ["run", "https://shop.example.com", "--i-own-this", + "--dir", _dir(tmp_path, SHOP)]) + assert res.exit_code == 2 + assert "--allow-writes shop.example.com" in _out(res) + + +def test_run_stops_on_a_broken_journey_file(tmp_path): + res = CliRunner().invoke(cli, ["run", "http://localhost:1", + "--dir", _dir(tmp_path, {**BROWSE, "version": 9})]) + assert res.exit_code == 2 + assert "version must be 1" in _out(res) + + +def test_run_journey_flag_conflicts_with_url_mode(tmp_path): + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--urls", "--journey", "x", + "--dir", _dir(tmp_path, BROWSE)]) + assert res.exit_code == 2 + assert "--journey can't be combined" in _out(res) + + +def test_run_unknown_journey(tmp_path): + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--journey", "nope", + "--dir", _dir(tmp_path, BROWSE)]) + assert res.exit_code == 2 + assert "no journey named nope" in _out(res) From 1cad16dd8fbf1792798757a239d7ebcbb732aa8e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 14:34:51 +0000 Subject: [PATCH 04/11] fix(journeys): cookies across redirects, pure --json, no / probe, honest think time and compare --- cli/src/prescale_cli/commands/run.py | 14 +-- cli/src/prescale_cli/compare.py | 25 +++++- cli/src/prescale_cli/journeys/load.py | 10 ++- cli/src/prescale_cli/journeys/replay.py | 38 +++++++-- cli/src/prescale_cli/loadtest.py | 52 +++++++----- cli/tests/test_compare.py | 22 +++++ cli/tests/test_journeys_load.py | 108 ++++++++++++++++++++++++ 7 files changed, 230 insertions(+), 39 deletions(-) diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index df01f9c..078b832 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -324,6 +324,8 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa if not as_json: console.print(f"\n[bold]PreScale[/bold] — replaying {len(journeys)} " f"journey{'s' if len(journeys) != 1 else ''} against [cyan]{url}[/cyan]") + # --json keeps stdout for the Result alone; notes about journeys go to stderr. + out = Console(stderr=True) if as_json else console checks = asyncio.run(_check_all(journeys, url, timeout=timeout, headers=headers)) runs = [JourneyRun(j, c) for j, c in zip(journeys, checks)] levels = default_levels(config["max_users"]) @@ -333,7 +335,7 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa if not run.check.ok: failed = run.check.failed what = "stale" if run.check.stale else "couldn't run" - console.print(f"[yellow]⚠ {name}: {what}, not load-tested[/yellow] " + out.print(f"[yellow]⚠ {name}: {what}, not load-tested[/yellow] " f"[dim]({escape(failed.step.name)}: {escape(failed.problem or '')})" "[/dim]", highlight=False) continue @@ -347,10 +349,10 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa stages, run.warning = asyncio.run(ramp( run.journey, url, levels=levels, stage_seconds=config["stage_seconds"], timeout=timeout, max_rps=config["max_rps"], warmup=config["warmup"], - repeat=config["repeat"], headers=headers, + repeat=config["repeat"], headers=headers, think=config["think_time_s"], progress_cb=progress.starting, on_stage=progress.finished)) except LoadError as exc: - console.print(f"[red]Error:[/red] {exc}") + out.print(f"[red]Error:[/red] {exc}") raise SystemExit(1) run.report = analyze(stages, latency_wall=latency_wall, error_threshold=error_threshold, @@ -358,7 +360,7 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa worst = first_to_break(runs) if worst is None: - console.print("[red]Error:[/red] no journey replays against this app, so there's " + out.print("[red]Error:[/red] no journey replays against this app, so there's " "nothing to load-test. Run `prescale journeys verify` for details.") raise SystemExit(1) warnings = list(dict.fromkeys(r.warning for r in runs if r.warning)) @@ -366,7 +368,9 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa targets=[s.label for s in worst.journey.steps], config={**config, "mode": "journeys"}, warning=" ".join(warnings) or None, replica=replica) - result["verdict"]["culprit_journey"] = worst.journey.name + # Only a journey that broke is a culprit; if all held, there is none. + result["verdict"]["culprit_journey"] = (worst.journey.name + if worst.report.onset_users is not None else None) result["journeys"] = journeys_block(runs) return worst.report, result, live_mode diff --git a/cli/src/prescale_cli/compare.py b/cli/src/prescale_cli/compare.py index d2d1a45..dea8b2e 100644 --- a/cli/src/prescale_cli/compare.py +++ b/cli/src/prescale_cli/compare.py @@ -27,6 +27,8 @@ def _facts(result: dict) -> dict: "branch": env.get("git_branch"), "created_at": result.get("created_at"), "target": result.get("target", {}).get("url"), + "mode": (result.get("config") or {}).get("mode") or "urls", + "culprit_journey": v.get("culprit_journey"), } @@ -35,6 +37,9 @@ def compare_results(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHO survives_users; it's 'confident' when the M1 bands don't overlap. A new run that held to its max below the old level is 'inconclusive', not a drop.""" o, n = _facts(old), _facts(new) + if o["mode"] != n["mode"]: + # URL capacity and journey capacity measure different things: never a regression. + return _not_comparable(o, n) o_surv, n_surv = o["survives_users"], n["survives_users"] pct = (n_surv - o_surv) / o_surv if o_surv else 0.0 inconclusive = (n["onset_users"] is None and n["max_tested"] is not None @@ -57,10 +62,28 @@ def compare_results(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHO "improved": improved, "inconclusive": inconclusive, "confident": confident, - "summary": _summary(o, n, pct, regressed, improved, inconclusive), + "summary": _summary(o, n, pct, regressed, improved, inconclusive) + _journey_note(o, n), } +def _not_comparable(o: dict, n: dict) -> dict: + kinds = {"urls": "tested URLs", "journeys": "replayed journeys"} + return { + "old": o, "new": n, "survives_delta": 0, "survives_pct": 0.0, "peak_rps_pct": 0.0, + "regressed": False, "improved": False, "inconclusive": True, "confident": False, + "summary": (f"Not comparable: the old run {kinds.get(o['mode'], o['mode'])}, the new " + f"run {kinds.get(n['mode'], n['mode'])}. Compare runs of the same kind."), + } + + +def _journey_note(o: dict, n: dict) -> str: + if o["culprit_journey"] and n["culprit_journey"] \ + and o["culprit_journey"] != n["culprit_journey"]: + return (f" The journey that breaks first changed: {o['culprit_journey']} → " + f"{n['culprit_journey']}.") + return "" + + def _summary(o: dict, n: dict, pct: float, regressed: bool, improved: bool, inconclusive: bool = False) -> str: arrow = f"{o['survives_users']} → {n['survives_users']}" diff --git a/cli/src/prescale_cli/journeys/load.py b/cli/src/prescale_cli/journeys/load.py index e398ecf..d25554a 100644 --- a/cli/src/prescale_cli/journeys/load.py +++ b/cli/src/prescale_cli/journeys/load.py @@ -35,16 +35,18 @@ def status(self) -> str: async def ramp(journey: Journey, base_url: str, *, levels: list[int], stage_seconds: float, timeout: float = 10.0, max_rps: float | None = None, warmup: bool = True, repeat: int = 1, headers: dict[str, str] | None = None, - env: dict[str, str] | None = None, progress_cb=None, on_stage=None, - transport: httpx.AsyncBaseTransport | None = None, + env: dict[str, str] | None = None, think: float = 0.0, progress_cb=None, + on_stage=None, transport: httpx.AsyncBaseTransport | None = None, ) -> tuple[list[StageResult], str | None]: - """Ramp virtual users who each walk JOURNEY, through LEVELS.""" + """Ramp virtual users who each walk JOURNEY, through LEVELS. THINK is the + pause for steps without their own `think`.""" return await run_loadtest( [base_url], levels=levels, stage_seconds=stage_seconds, timeout=timeout, max_rps=max_rps, warmup=warmup, repeat=repeat, headers=headers, host_permission=True, # the caller has already checked the host's rules progress_cb=progress_cb, on_stage=on_stage, transport=transport, - vu=journey_vu(journey, base_url, env), client_options=client_options(base_url), + vu=journey_vu(journey, base_url, env, think), client_options=client_options(base_url), + preflight=False, # `check` already replayed the journey once ) diff --git a/cli/src/prescale_cli/journeys/replay.py b/cli/src/prescale_cli/journeys/replay.py index 69e35a3..4760ff3 100644 --- a/cli/src/prescale_cli/journeys/replay.py +++ b/cli/src/prescale_cli/journeys/replay.py @@ -29,6 +29,7 @@ from prescale_cli.loadtest import client_headers _SNIPPET = 160 +_MAX_REDIRECTS = 10 class OffOriginError(Exception): @@ -117,12 +118,14 @@ async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, return result -def journey_vu(journey: Journey, base_url: str, env: dict[str, str] | None = None): +def journey_vu(journey: Journey, base_url: str, env: dict[str, str] | None = None, + think: float = 0.0): """JOURNEY as a load-test virtual user: walk its steps, pausing for each step's think time, and start again with a fresh identity until the stage deadline. A failed step ends that walk (after its pause), like a real user giving up. Each step is recorded under its label, so the report can name the - step that broke.""" + step that broke. THINK is the pause for steps that don't set their own + (`--think-time`, or a launch profile's).""" env = dict(os.environ) if env is None else env async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> None: @@ -139,13 +142,16 @@ async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> Non _record(sink, outcome) # A user pauses after a step either way; restarting the instant a step # fails would hammer the app harder than real users who hit an error. - if step.think: + pause = step.think or think + if pause: remaining = deadline - loop.time() if remaining <= 0: return - await asyncio.sleep(min(step.think, remaining)) + await asyncio.sleep(min(pause, remaining)) if not outcome.ok: break + # Never spin: a walk that failed without awaiting must still yield. + await asyncio.sleep(0) return vu @@ -179,10 +185,9 @@ async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, request = client.build_request(step.method, url, headers=req_headers, json=payload, content=body) - jar.set_cookie_header(request) start = time.perf_counter() try: - resp = await client.send(request) + resp = await _send(client, request, jar) except OffOriginError as exc: outcome.problem, outcome.kind = str(exc), "left the app" return outcome @@ -200,8 +205,6 @@ async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, outcome.latency = time.perf_counter() - start outcome.status = resp.status_code outcome.headers = resp.headers - for hop in (*resp.history, resp): - jar.extract_cookies(hop) if not step.accepts(resp.status_code): wanted = (" or ".join(map(str, step.expect)) if step.expect else "a status below 400") @@ -222,6 +225,25 @@ async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, return outcome +async def _send(client: httpx.AsyncClient, request: httpx.Request, + jar: httpx.Cookies) -> httpx.Response: + """Send REQUEST with this run's cookies, following redirects one hop at a time + so every hop carries the cookies earlier hops set (a login that redirects to + /account stays logged in). The client keeps no cookies of its own, and its + same-origin hook checks each hop.""" + for _ in range(_MAX_REDIRECTS + 1): + if "cookie" in request.headers: + del request.headers["cookie"] + jar.set_cookie_header(request) + resp = await client.send(request, follow_redirects=False) + jar.extract_cookies(resp) + if resp.next_request is None: + return resp + await resp.aclose() + request = resp.next_request + raise httpx.TooManyRedirects("too many redirects", request=request) + + def extract(resp: httpx.Response, source: str) -> Any: """Read `body.a.b.0` (JSON) or `header.Name` from a response; None if missing.""" kind, _, where = source.partition(".") diff --git a/cli/src/prescale_cli/loadtest.py b/cli/src/prescale_cli/loadtest.py index 8005efb..131d9f7 100644 --- a/cli/src/prescale_cli/loadtest.py +++ b/cli/src/prescale_cli/loadtest.py @@ -495,6 +495,7 @@ async def run_loadtest( host_permission: bool = False, vu=None, client_options: dict | None = None, + preflight: bool = True, ) -> tuple[list[StageResult], str | None]: """Preflight the target, then ramp through `levels`, spreading each stage's load across `targets`. Stops early once a stage is more than `hard_stop_rate` @@ -504,7 +505,9 @@ async def run_loadtest( after the preflight unless `host_permission` says the user has permission. `vu` replaces the default virtual user (see `_run_stage`), and - `client_options` adds settings to the HTTP client (journeys use both).""" + `client_options` adds settings to the HTTP client. Journeys use both, and skip + the preflight (`preflight=False`): each journey was already replayed once, which + checks the app better than a request to targets[0].""" if not targets: raise LoadError("No targets to test.") max_conns = max(levels) + 50 @@ -516,26 +519,8 @@ async def run_loadtest( timeout=timeout, limits=limits, follow_redirects=True, headers=client_headers(headers), transport=transport, **(client_options or {}), ) as client: - try: - preflight = await client.request(method, targets[0]) - except httpx.HTTPError as exc: - raise LoadError(f"Couldn't reach {targets[0]}: {exc}") from exc - blocker = edge_blocker(preflight.headers) - if blocker: - # Load now would only measure the firewall, and deepen the block. - raise LoadError( - f"{_capitalize(blocker)} is blocking requests from this machine (it answered " - f"the first request with HTTP {preflight.status_code}), so a load test would " - f"only measure the firewall. {_INSTEAD}" - ) - rule = detect_host(preflight.headers) - if rule and not host_permission: - raise HostPolicyError(rule) - if preflight.status_code >= 400: - warning = ( - f"First request returned HTTP {preflight.status_code} — " - "results may reflect a broken endpoint, not a load limit." - ) + if preflight: + warning = await _preflight(client, method, targets[0], host_permission) if warmup: # Discarded: warms caches/JIT/connection pools so the first measured @@ -563,6 +548,31 @@ async def run_loadtest( return stages, warning +async def _preflight(client: httpx.AsyncClient, method: str, target: str, + host_permission: bool) -> str | None: + """One request before any load: stop if a firewall already blocks us or the + host forbids load tests; return a warning if the target answers with an error.""" + try: + first = await client.request(method, target) + except httpx.HTTPError as exc: + raise LoadError(f"Couldn't reach {target}: {exc}") from exc + blocker = edge_blocker(first.headers) + if blocker: + # Load now would only measure the firewall, and deepen the block. + raise LoadError( + f"{_capitalize(blocker)} is blocking requests from this machine (it answered " + f"the first request with HTTP {first.status_code}), so a load test would " + f"only measure the firewall. {_INSTEAD}" + ) + rule = detect_host(first.headers) + if rule and not host_permission: + raise HostPolicyError(rule) + if first.status_code >= 400: + return (f"First request returned HTTP {first.status_code} — " + "results may reflect a broken endpoint, not a load limit.") + return None + + def open_client(*, timeout: float = 10.0, max_conns: int = 64, transport: httpx.AsyncBaseTransport | None = None, headers: dict[str, str] | None = None) -> httpx.AsyncClient: diff --git a/cli/tests/test_compare.py b/cli/tests/test_compare.py index 2f4a97c..4dd404a 100644 --- a/cli/tests/test_compare.py +++ b/cli/tests/test_compare.py @@ -103,3 +103,25 @@ def test_held_run_below_the_old_level_is_inconclusive_not_a_regression(): def test_held_run_that_reached_the_old_level_still_compares(): c = compare_results(_result(200, low=180, high=220), _result(190, held_at=190)) assert not c["inconclusive"] and not c["regressed"] + + +def _mode(result, mode=None, culprit=None): + r = json.loads(json.dumps(result)) + if mode: + r["config"]["mode"] = mode + r["verdict"]["culprit_journey"] = culprit + return r + + +def test_url_run_and_journey_run_are_not_comparable(): + old = _result(200) + new = _mode(_result(20), mode="journeys", culprit="checkout") + cmp = compare_results(old, new) + assert not cmp["regressed"] and cmp["inconclusive"] + assert cmp["summary"].startswith("Not comparable") + + +def test_journey_runs_note_a_new_first_to_break(): + old = _mode(_result(100), mode="journeys", culprit="search") + new = _mode(_result(100), mode="journeys", culprit="checkout") + assert "search → checkout" in compare_results(old, new)["summary"] diff --git a/cli/tests/test_journeys_load.py b/cli/tests/test_journeys_load.py index 2c7897f..6d90c15 100644 --- a/cli/tests/test_journeys_load.py +++ b/cli/tests/test_journeys_load.py @@ -150,3 +150,111 @@ def test_run_unknown_journey(tmp_path): "--dir", _dir(tmp_path, BROWSE)]) assert res.exit_code == 2 assert "no journey named nope" in _out(res) + + +# --- review fixes ------------------------------------------------------------- + +LOGIN_REDIRECT = { + "version": 1, "name": "account", "type": "account", + "steps": [ + {"name": "log in", "request": {"method": "POST", "path": "/login"}, + "expect": {"status": 200}, "extract": {"user": "body.user"}}, + {"name": "update cart", "request": {"method": "POST", "path": "/cart"}, + "expect": {"status": 200}, "extract": {"again": "body.user"}}, + ], +} + + +def _login_redirects(): + """POST /login sets a cookie on a 302; POST /cart answers 303. The pages they + redirect to only know the user if the cookie came along.""" + logins = iter(range(10_000)) + + def handler(request): + cookie = request.headers.get("cookie") + if request.url.path == "/login": + return httpx.Response(302, headers={"location": "/me", + "set-cookie": f"sid=u{next(logins)}; Path=/"}) + if request.url.path == "/cart": + return httpx.Response(303, headers={"location": "/cartview"}) + if request.url.path in ("/me", "/cartview"): + return httpx.Response(200, json={"user": cookie} if cookie else {}) + if request.url.path == "/": + return httpx.Response(302, headers={"location": "https://sso.example.com/"}) + return httpx.Response(404) + + return httpx.MockTransport(handler) + + +def test_cookies_survive_redirects_in_verify(): + result = asyncio.run(check(parse(LOGIN_REDIRECT), "http://shop.test", env={}, + transport=_login_redirects())) + assert result.ok, result.failed and result.failed.problem + + +def test_cookies_survive_redirects_at_load_and_root_redirect_is_fine(): + # "/" redirects to an SSO host: journeys don't probe it, so no OffOriginError. + stages, warning = _ramp(parse(LOGIN_REDIRECT), _login_redirects(), levels=(1, 5)) + assert warning is None + for stage in stages: + assert all(stat.errors == 0 for stat in stage.routes.values()), stage.routes + + +def _fake_checks(monkeypatch, stale_names=()): + """Replace the network in `run`'s journey mode: CHECK says which journeys are + stale; RAMP returns a run that holds at every level.""" + from prescale_cli.commands import run as run_cmd + from prescale_cli.journeys.replay import JourneyCheck, StepCheck + from prescale_cli.loadtest import RouteStat, StageResult + + async def fake_check_all(journeys, url, *, timeout, headers): + out = [] + for j in journeys: + c = JourneyCheck(j) + ok = j.name not in stale_names + c.steps = [StepCheck(s, status=200 if ok else 404, ok=ok, + problem=None if ok else "returned 404, expected 200") + for s in j.steps[:1]] if not ok else \ + [StepCheck(s, status=200, ok=True) for s in j.steps] + out.append(c) + return out + + async def fake_ramp(journey, url, *, levels, **kwargs): + routes = {s.label: RouteStat(total=10, latencies=[0.01] * 10) for s in journey.steps} + return [StageResult(users=u, duration=1.0, routes=dict(routes)) for u in levels], None + + monkeypatch.setattr(run_cmd, "_check_all", fake_check_all) + monkeypatch.setattr(run_cmd, "ramp", fake_ramp) + + +def test_run_json_stays_pure_when_a_journey_is_stale(tmp_path, monkeypatch): + _fake_checks(monkeypatch, stale_names={"browse"}) + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--json", "--no-save", + "-u", "5", "--dir", _dir(tmp_path, SHOP, BROWSE)]) + assert res.exit_code == 0, res.output + result = json.loads(res.stdout) # nothing but the Result on stdout + assert "stale, not load-tested" in res.stderr + statuses = {j["name"]: j["status"] for j in result["journeys"]} + assert statuses == {"browse": "stale", "checkout": "ok"} + # nothing broke, so there's no culprit journey + assert result["verdict"]["onset_users"] is None + assert result["verdict"]["culprit_journey"] is None + assert result["config"]["mode"] == "journeys" + + +def test_run_applies_think_time_to_steps_without_their_own(tmp_path, monkeypatch): + seen = {} + _fake_checks(monkeypatch) + from prescale_cli.commands import run as run_cmd + inner = run_cmd.ramp + + async def spy(journey, url, **kwargs): + seen[journey.name] = kwargs.get("think") + return await inner(journey, url, **kwargs) + + monkeypatch.setattr(run_cmd, "ramp", spy) + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--json", "--no-save", + "-u", "5", "--think-time", "1.5", + "--dir", _dir(tmp_path, BROWSE)]) + assert res.exit_code == 0, res.output + assert seen == {"browse": 1.5} From 92b0911e3fd9364b38519eab4714bf5604ec9161 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 15:32:43 +0000 Subject: [PATCH 05/11] feat(journeys): ramp journeys together to catch interference, and --expect --- CHANGELOG.md | 7 ++ README.md | 9 ++ cli/src/prescale_cli/commands/run.py | 88 ++++++++++++++- cli/src/prescale_cli/journeys/load.py | 62 ++++++++++- cli/src/prescale_cli/journeys/replay.py | 34 ++++-- cli/src/prescale_cli/render.py | 49 +++++++++ .../prescale_cli/schema/result.schema.json | 37 +++++++ cli/tests/test_journeys_load.py | 104 +++++++++++++++++- 8 files changed, 367 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6204b14..f7e87fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,13 @@ break first, with per-step timings, and saved runs carry a `journeys` field (plus `verdict.culprit_journey`). Stale journeys are skipped with the reason. `--journey` picks journeys; `--urls` tests URLs as before. +- **Journeys together, and interference.** After ramping each journey alone, `prescale run` + ramps them all together, each user picking a journey by `weight`, and flags a journey that + breaks with fewer of its own users than it held alone: it competes with the others for + something they share (a database pool, CPU, a cache). Saved runs carry a `combined` field. + `--no-combined` skips it. +- **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split + across journeys by weight: each must hold its share alone, and all together must hold N. ### Fixed - **`prescale mcp` works on a fresh install again.** `pip install 'prescale[mcp]'` picked up the new diff --git a/README.md b/README.md index 16371a8..2caccce 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,15 @@ When `prescale/journeys/` has journeys, **`prescale run` replays them at load**: A stale journey isn't load-tested (a failure with no load says nothing about capacity); it's listed with the reason. `--journey NAME` picks journeys, and `--urls` (or `--path`, `--from-sitemap`) tests URLs instead. +Then **all journeys run together**, each user picking a journey by its `weight` (defaults by type: browse 60, search 25, account 8, checkout 5, write 5, auth 2; edit them in the files). That catches **interference**: journeys competing for something they share. If a journey breaks with fewer of its own users than it held alone, the report says so: + +``` +Interference checkout broke with only ~15 of its own users, but held ~20 alone. + It competes with quickbuy for something they share (a database pool, CPU, a cache). +``` + +`--no-combined` skips this pass. `--expect 500` says whether you'd hold 500 users at peak: in a journey run, each journey must hold its share of them (by weight) on its own, and all journeys together must hold 500. + - `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. - **Secrets never go in the file.** A literal password, token, API key or `Authorization` value is refused; use `{{env.NAME}}`. - Paths start with `/` and are relative to the URL you give (`http://host/app` + `/api/cart` is `http://host/app/api/cart`). A journey only calls that app: a request or redirect to another host fails the step. diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index 078b832..90bb7bc 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -21,7 +21,13 @@ from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica from prescale_cli.commands.upload import header_option, upload_after_run, upload_option -from prescale_cli.journeys.load import JourneyRun, first_to_break, journeys_block, ramp +from prescale_cli.journeys.load import ( + JourneyRun, + combined_block, + first_to_break, + journeys_block, + ramp, +) from prescale_cli.journeys.model import Journey, load_dir from prescale_cli.journeys.replay import check from prescale_cli.live import LiveRamp, ramp_progress @@ -89,6 +95,10 @@ help="Frame the run as a launch scenario (see `prescale profiles`).") @click.option("--journey", "only", multiple=True, help="Replay only this journey (repeatable). See `prescale journeys`.") +@click.option("--no-combined", "no_combined", is_flag=True, + help="Skip ramping all journeys together (by weight) after each one alone.") +@click.option("--expect", "expect", default=None, type=click.IntRange(min=1), metavar="N", + help="Users you expect at peak; the verdict says whether you'd hold them.") @click.option("--urls", "urls_only", is_flag=True, help="Test URLs, not journeys, even when prescale/journeys/ has some.") @click.option("--allow-writes", "allow_writes", multiple=True, metavar="HOST", @@ -106,7 +116,8 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, repeat: int, think_time: float, ignore_robots: bool, yes: bool, as_json: bool, html_path: str | None, store: str | None, no_save: bool, fail_under: int | None, - profile_name: str | None, only: tuple[str, ...], urls_only: bool, + profile_name: str | None, only: tuple[str, ...], no_combined: bool, + expect: int | None, urls_only: bool, allow_writes: tuple[str, ...], directory: Path, app: str | None, app_timeout: float, host_permission: bool, headers: dict[str, str], upload: bool) -> None: @@ -190,13 +201,13 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, "profile": prof.name if prof else None, } finish = dict(prof=prof, no_save=no_save, store=store, html_path=html_path, - as_json=as_json, upload=upload, fail_under=fail_under) + as_json=as_json, upload=upload, fail_under=fail_under, expect=expect) if journeys: report, result, live_mode = _run_journeys( journeys, url, config, latency_wall=latency_wall, error_threshold=error_threshold, timeout=timeout, headers=headers, - replica=bool(app), as_json=as_json) + replica=bool(app), as_json=as_json, combined=not no_combined) _finish(result, report, live_mode=live_mode, **finish) return @@ -259,8 +270,10 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, def _finish(result: dict, report, *, prof, no_save: bool, store: str | None, html_path: str | None, as_json: bool, upload: bool, fail_under: int | None, - live_mode: bool) -> None: + live_mode: bool, expect: int | None = None) -> None: """Save, render, upload and gate a finished run (URLs or journeys alike).""" + if expect is not None: + result["expect"] = expect_block(result, expect) if prof is not None: result["profile"] = scenario_block(prof, report.survives_users) saved_path = None if no_save else write_result(result, store=store) @@ -288,6 +301,46 @@ def _finish(result: dict, report, *, prof, no_save: bool, store: str | None, raise SystemExit(1) +def _meets(survives: int, held_all: bool, users: float) -> bool | None: + """True if it held USERS; None if it held everything tested but we never + tested that many; False if it broke below them.""" + if survives >= users: + return True + return None if held_all else False + + +def expect_block(result: dict, users: int) -> dict: + """Whether the run holds the USERS you expect at peak. + + A URL run compares its verdict. A journey run splits USERS across journeys + by weight: each journey must hold its share on its own, and all journeys + together (if they were ramped together) must hold USERS. It meets the + expectation only if every one of those does.""" + v = result["verdict"] + journeys = [j for j in (result.get("journeys") or []) if j["status"] == "ok"] + if not journeys: + return {"users": users, "meets": _meets(v["survives_users"], + v["onset_users"] is None, users), + "survives_users": v["survives_users"], "journeys": None, "combined": None} + total = sum(j["weight"] for j in journeys) + per = [] + for j in journeys: + share = users * j["weight"] / total + per.append({"name": j["name"], "users": round(share, 1), + "survives_users": j["survives_users"], + "meets": _meets(j["survives_users"], j["onset_users"] is None, share)}) + checks = [p["meets"] for p in per] + combined = None + if result.get("combined"): + c = result["combined"] + combined = {"survives_users": c["survives_users"], + "meets": _meets(c["survives_users"], c["onset_users"] is None, users)} + checks.append(combined["meets"]) + meets = False if False in checks else (None if None in checks else True) + return {"users": users, "meets": meets, "survives_users": v["survives_users"], + "journeys": per, "combined": combined} + + def _journeys_to_run(directory: Path, only: tuple[str, ...], *, urls_only: bool, url_mode: bool) -> list[Journey]: """The journeys this run replays: none in URL mode (--urls, --path, @@ -317,7 +370,7 @@ def _journeys_to_run(directory: Path, only: tuple[str, ...], *, urls_only: bool, def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wall: float, error_threshold: float, timeout: float, headers: dict[str, str], - replica: bool, as_json: bool) -> tuple: + replica: bool, as_json: bool, combined: bool = True) -> tuple: """The isolation pass: check each journey once, then ramp each one that replays on its own. Returns (report, result, live_mode) for the first journey to break, with every journey's verdict in result["journeys"].""" @@ -372,6 +425,29 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa result["verdict"]["culprit_journey"] = (worst.journey.name if worst.report.onset_users is not None else None) result["journeys"] = journeys_block(runs) + + ramped = [r for r in runs if r.report is not None] + if combined and len(ramped) > 1: + if not as_json: + mix = ", ".join(f"{escape(r.journey.name)} {r.journey.weight:g}" for r in ramped) + console.print(f"\n[bold]All journeys together[/bold] [dim](by weight: {mix})[/dim]", + highlight=False) + progress = ramp_progress(console, latency_wall=latency_wall, as_json=as_json) + live_mode = isinstance(progress, LiveRamp) + try: + with progress: + stages, _ = asyncio.run(ramp( + [r.journey for r in ramped], url, levels=levels, + stage_seconds=config["stage_seconds"], timeout=timeout, + max_rps=config["max_rps"], warmup=config["warmup"], + repeat=config["repeat"], headers=headers, think=config["think_time_s"], + progress_cb=progress.starting, on_stage=progress.finished)) + except LoadError as exc: + out.print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) + together = analyze(stages, latency_wall=latency_wall, error_threshold=error_threshold, + rate_capped=config["max_rps"] is not None) + result["combined"] = combined_block(together, runs) return worst.report, result, live_mode diff --git a/cli/src/prescale_cli/journeys/load.py b/cli/src/prescale_cli/journeys/load.py index d25554a..139c064 100644 --- a/cli/src/prescale_cli/journeys/load.py +++ b/cli/src/prescale_cli/journeys/load.py @@ -1,10 +1,14 @@ -"""Load-test journeys: the isolation pass. +"""Load-test journeys: the isolation pass, then the combined pass. Each journey is first replayed once (`check`). A journey that fails there is stale or unreachable and isn't ramped: a failure with no load says nothing about capacity. Every journey that replays is then ramped on its own, so its verdict (how many users it survives, the step that breaks first, why) doesn't depend on any traffic mix. The run's overall verdict is the first journey to break. + +The combined pass then ramps all of them together, each user picking a journey +by `weight`, to catch interference: journeys competing for something they share +(a database pool, CPU, a cache), which no journey shows on its own. """ from __future__ import annotations @@ -14,7 +18,7 @@ import httpx from prescale_cli.journeys.model import Journey -from prescale_cli.journeys.replay import JourneyCheck, client_options, journey_vu +from prescale_cli.journeys.replay import SEP, JourneyCheck, client_options, journey_vu from prescale_cli.loadtest import RunReport, StageResult, run_loadtest @@ -32,14 +36,16 @@ def status(self) -> str: return "stale" if self.check.stale else "unreachable" -async def ramp(journey: Journey, base_url: str, *, levels: list[int], stage_seconds: float, +async def ramp(journey: Journey | list[Journey], base_url: str, *, levels: list[int], + stage_seconds: float, timeout: float = 10.0, max_rps: float | None = None, warmup: bool = True, repeat: int = 1, headers: dict[str, str] | None = None, env: dict[str, str] | None = None, think: float = 0.0, progress_cb=None, on_stage=None, transport: httpx.AsyncBaseTransport | None = None, ) -> tuple[list[StageResult], str | None]: - """Ramp virtual users who each walk JOURNEY, through LEVELS. THINK is the - pause for steps without their own `think`.""" + """Ramp virtual users who each walk JOURNEY, through LEVELS; given a list, + each walk picks one by weight (the combined pass). THINK is the pause for + steps without their own `think`.""" return await run_loadtest( [base_url], levels=levels, stage_seconds=stage_seconds, timeout=timeout, max_rps=max_rps, warmup=warmup, repeat=repeat, headers=headers, @@ -122,3 +128,49 @@ def _step_stats(index: int, step, stage: StageResult | None) -> dict: "p95_ms": round(stat.pct(0.95) * 1000) if has_latency else None, "error_rate": round(stat.error_rate, 4), } + + +def combined_block(report: RunReport, runs: list[JourneyRun]) -> dict: + """The Result's `combined` field: the journeys ramped together by weight. + Users here are the total across journeys; each journey gets its weight's share. + If the journey that breaks first does so with fewer of its own users than it + held alone, something it shares with the others is the limit (interference).""" + ramped = [r for r in runs if r.report is not None] + total = sum(r.journey.weight for r in ramped) + share = {r.journey.name: r.journey.weight / total for r in ramped} + block: dict = { + "journeys": [r.journey.name for r in ramped], + "mix": {name: round(part * 100, 1) for name, part in share.items()}, + "survives_users": report.survives_users, + "max_tested": report.max_tested, + "onset_users": report.onset_users, + "onset_reason": report.onset_reason, + "confidence": {"survives_low": report.survives_low, + "survives_high": report.survives_high, "stable": report.stable}, + "culprit_journey": None, + "culprit_step": None, + "bottleneck": report.bottleneck, + "interference": None, + } + if report.onset_users is None or not report.culprit_route or SEP not in report.culprit_route: + return block + name, label = report.culprit_route.split(SEP, 1) + run = next((r for r in ramped if r.journey.name == name), None) + if run is None: + return block + index = next((i for i, s in enumerate(run.journey.steps, 1) if s.label == label), None) + block["culprit_journey"] = name + if index is not None: + block["culprit_step"] = {"index": index, "name": run.journey.steps[index - 1].name, + "route": label} + users_of_it = report.onset_users * share[name] + alone = (run.report.survives_users if run.report.onset_users is not None + else run.report.max_tested) + if users_of_it < alone: + block["interference"] = { + "journey": name, + "users_at_break": round(users_of_it, 1), + "held_alone": alone, + "with": [r.journey.name for r in ramped if r.journey.name != name], + } + return block diff --git a/cli/src/prescale_cli/journeys/replay.py b/cli/src/prescale_cli/journeys/replay.py index 4760ff3..a79e9b3 100644 --- a/cli/src/prescale_cli/journeys/replay.py +++ b/cli/src/prescale_cli/journeys/replay.py @@ -18,6 +18,7 @@ import asyncio import os +import random import time from dataclasses import dataclass, field from http.cookiejar import CookieJar, DefaultCookiePolicy @@ -29,6 +30,8 @@ from prescale_cli.loadtest import client_headers _SNIPPET = 160 +# Separates journey and step in labels when several journeys run together. +SEP = " › " _MAX_REDIRECTS = 10 @@ -118,19 +121,28 @@ async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, return result -def journey_vu(journey: Journey, base_url: str, env: dict[str, str] | None = None, - think: float = 0.0): - """JOURNEY as a load-test virtual user: walk its steps, pausing for each - step's think time, and start again with a fresh identity until the stage - deadline. A failed step ends that walk (after its pause), like a real user - giving up. Each step is recorded under its label, so the report can name the - step that broke. THINK is the pause for steps that don't set their own - (`--think-time`, or a launch profile's).""" +def journey_vu(journeys: Journey | list[Journey], base_url: str, + env: dict[str, str] | None = None, think: float = 0.0, + rng: random.Random | None = None): + """A load-test virtual user: walk a journey's steps, pausing for each step's + think time, and start again with a fresh identity until the stage deadline. + A failed step ends that walk (after its pause), like a real user giving up. + + Given several JOURNEYS, each walk picks one by `weight`, so they share the + app the way real traffic does; steps are then labelled `journey › step`. + Each step is recorded under its label, so the report can name the step that + broke. THINK is the pause for steps that don't set their own (`--think-time`, + or a launch profile's).""" env = dict(os.environ) if env is None else env + mix = [journeys] if isinstance(journeys, Journey) else list(journeys) + weights = [j.weight for j in mix] + rng = rng or random.Random() async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> None: loop = asyncio.get_running_loop() while loop.time() < deadline: + journey = mix[0] if len(mix) == 1 else rng.choices(mix, weights)[0] + prefix = f"{journey.name}{SEP}" if len(mix) > 1 else "" variables: dict[str, Any] = fresh_uniques() jar = httpx.Cookies() for step in journey.steps: @@ -139,7 +151,7 @@ async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> Non if loop.time() >= deadline: return outcome = await run_step(client, step, base_url, variables, env, jar) - _record(sink, outcome) + _record(sink, outcome, prefix) # A user pauses after a step either way; restarting the instant a step # fails would hammer the app harder than real users who hit an error. pause = step.think or think @@ -156,10 +168,10 @@ async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> Non return vu -def _record(sink, outcome: StepCheck) -> None: +def _record(sink, outcome: StepCheck, prefix: str = "") -> None: """One step's outcome into the stage's numbers. An answer the journey didn't expect counts as a failure, even a 4xx: under load that's a step going wrong.""" - label = outcome.step.label + label = prefix + outcome.step.label if outcome.status is None: sink.record_error(label, outcome.kind or "network") elif outcome.ok or outcome.status >= 500 or outcome.status == 429: diff --git a/cli/src/prescale_cli/render.py b/cli/src/prescale_cli/render.py index b7dafb3..887d671 100644 --- a/cli/src/prescale_cli/render.py +++ b/cli/src/prescale_cli/render.py @@ -129,6 +129,10 @@ def render_terminal(result: dict, *, show_ramp: bool = True) -> None: bar = text_bar(verdict["survives_users"], scale, width=24, marker=prof["peak_users"]) lines.append(f" [{tone}]{bar}[/{tone}] ┆ = launch peak") + exp = result.get("expect") + if exp: + lines.extend(_expect_lines(exp)) + console.print(Panel("\n".join(lines), title="📈 Readiness report", border_style=color)) if multi and stages: @@ -137,6 +141,27 @@ def render_terminal(result: dict, *, show_ramp: bool = True) -> None: render_journeys(result) +def _expect_lines(exp: dict) -> list[str]: + """The readiness lines for --expect: the answer first, then what decides it.""" + icon = {True: "✅", False: "🛑", None: "❔"}[exp["meets"]] + if not exp.get("journeys"): + tail = {True: f"survives ~{exp['survives_users']}.", + False: f"survives ~{exp['survives_users']}, short by " + f"~{exp['users'] - exp['survives_users']}.", + None: f"held all ~{exp['survives_users']} tested; raise --max-users to know."} + return [f"Expected {icon} ~{exp['users']} users: {tail[exp['meets']]}"] + lines = [f"Expected {icon} ~{exp['users']} users, split across journeys by weight:"] + for j in exp["journeys"]: + mark = {True: "✅", False: "🛑", None: "❔"}[j["meets"]] + lines.append(f" {mark} {escape(j['name'])}: ~{j['users']:g} users, holds " + f"~{j['survives_users']} alone") + c = exp.get("combined") + if c: + mark = {True: "✅", False: "🛑", None: "❔"}[c["meets"]] + lines.append(f" {mark} all together: holds ~{c['survives_users']}") + return lines + + def render_journeys(result: dict) -> None: """One row per journey: how many users it survives and the step that breaks first. Each was ramped on its own, so no row depends on the others.""" @@ -168,6 +193,30 @@ def render_journeys(result: dict) -> None: style = "bold red" if j["name"] == first and j["onset_users"] is not None else None table.add_row(name, survives, where, escape(j.get("bottleneck") or ""), style=style) console.print(table) + if result.get("combined"): + _render_combined(result["combined"]) + + +def _render_combined(c: dict) -> None: + """All journeys ramped together by weight, and whether one starved another.""" + mix = ", ".join(f"{escape(name)} {pct:g}%" for name, pct in c["mix"].items()) + lines = [f"[bold]Mix[/bold] {mix}"] + if c["onset_users"] is None: + lines.append(f"Held through {c['max_tested']} users in total.") + else: + step = c.get("culprit_step") + where = (f"{escape(c['culprit_journey'])} › {step['index']}. {escape(step['name'])}" + if c.get("culprit_journey") and step else "—") + lines.append(f"Survives ~{c['survives_users']} users in total; breaks first at {where}.") + inter = c.get("interference") + if inter: + others = ", ".join(escape(n) for n in inter["with"]) + lines.append( + f"[yellow]Interference[/yellow] {escape(inter['journey'])} broke with only " + f"~{inter['users_at_break']:g} of its own users, but held ~{inter['held_alone']} " + f"alone. It competes with {others} for something they share (a database pool, " + "CPU, a cache).") + console.print(Panel("\n".join(lines), title="All journeys together", border_style="cyan")) def _render_routes(result: dict) -> None: diff --git a/cli/src/prescale_cli/schema/result.schema.json b/cli/src/prescale_cli/schema/result.schema.json index e2e406e..f8a4072 100644 --- a/cli/src/prescale_cli/schema/result.schema.json +++ b/cli/src/prescale_cli/schema/result.schema.json @@ -98,6 +98,43 @@ "description": "Present on journey runs: one entry per journey. Each journey that replays is ramped on its own; a stale or unreachable one isn't load-tested and carries the reason.", "items": { "$ref": "#/$defs/journey" } }, + "combined": { + "type": "object", + "description": "Journey runs with two or more journeys that replay: all of them ramped together, each user picking a journey by weight. Users are the total across journeys.", + "properties": { + "journeys": { "type": "array", "items": { "type": "string" } }, + "mix": { "type": "object", "additionalProperties": { "type": "number" }, "description": "Percent of users per journey." }, + "survives_users": { "type": "integer" }, + "max_tested": { "type": "integer" }, + "onset_users": { "type": ["integer", "null"] }, + "onset_reason": { "type": ["string", "null"], "enum": ["errors", "latency", null] }, + "confidence": { "type": "object" }, + "culprit_journey": { "type": ["string", "null"] }, + "culprit_step": { "type": ["object", "null"] }, + "bottleneck": { "type": ["string", "null"] }, + "interference": { + "type": ["object", "null"], + "description": "Set when the journey that breaks first does so with fewer of its own users than it held alone: it competes with the others for something they share.", + "properties": { + "journey": { "type": "string" }, + "users_at_break": { "type": "number" }, + "held_alone": { "type": "integer" }, + "with": { "type": "array", "items": { "type": "string" } } + } + } + } + }, + "expect": { + "type": "object", + "description": "Present with `run --expect N`: whether the run holds N users at peak. In a journey run N is split across journeys by weight; each journey must hold its share alone, and the combined run must hold N.", + "properties": { + "users": { "type": "integer" }, + "meets": { "type": ["boolean", "null"], "description": "null: held every level tested, but fewer users than expected were tested." }, + "survives_users": { "type": "integer" }, + "journeys": { "type": ["array", "null"] }, + "combined": { "type": ["object", "null"] } + } + }, "investigation": { "type": "object", "description": "Present when `prescale investigate` diagnosed a failure.", diff --git a/cli/tests/test_journeys_load.py b/cli/tests/test_journeys_load.py index 6d90c15..aac61e2 100644 --- a/cli/tests/test_journeys_load.py +++ b/cli/tests/test_journeys_load.py @@ -220,7 +220,9 @@ async def fake_check_all(journeys, url, *, timeout, headers): return out async def fake_ramp(journey, url, *, levels, **kwargs): - routes = {s.label: RouteStat(total=10, latencies=[0.01] * 10) for s in journey.steps} + mix = journey if isinstance(journey, list) else [journey] + routes = {(f"{j.name} › " if len(mix) > 1 else "") + s.label: + RouteStat(total=10, latencies=[0.01] * 10) for j in mix for s in j.steps} return [StageResult(users=u, duration=1.0, routes=dict(routes)) for u in levels], None monkeypatch.setattr(run_cmd, "_check_all", fake_check_all) @@ -258,3 +260,103 @@ async def spy(journey, url, **kwargs): "--dir", _dir(tmp_path, BROWSE)]) assert res.exit_code == 0, res.output assert seen == {"browse": 1.5} + + +# --- the combined pass and --expect ------------------------------------------------ + +def test_mixed_users_pick_journeys_by_weight_and_label_steps_by_journey(): + import random + + from prescale_cli.journeys.replay import journey_vu + from prescale_cli.loadtest import _Sink + + heavy = parse({**BROWSE, "name": "heavy", "weight": 9}) + light = parse({**BROWSE, "name": "light", "weight": 1, + "steps": [{"request": {"path": "/light"}}]}) + vu = journey_vu([heavy, light], "http://shop.test", env={}, rng=random.Random(7)) + sink = _Sink(labels=False) + + async def go(): + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200))) as c: + await vu(c, asyncio.get_running_loop().time() + 0.2, sink) + asyncio.run(go()) + counts = {label: stat.total for label, stat in sink.routes.items()} + assert set(counts) == {"heavy › GET /", "light › GET /light"} + assert counts["heavy › GET /"] > 4 * counts["light › GET /light"] + + +def _report(survives, onset, culprit=None, max_tested=100): + from prescale_cli.loadtest import RunReport + return RunReport(stages=[], survives_users=survives, max_tested=max_tested, + onset_users=onset, onset_reason="errors" if onset else None, + culprit_route=culprit, survives_low=survives, survives_high=survives) + + +def test_combined_block_flags_interference(): + from prescale_cli.journeys.load import combined_block + checkout = parse({**SHOP, "weight": 1}) + search = parse({**BROWSE, "name": "search", "weight": 3}) + runs = [JourneyRun(checkout, None, report=_report(40, 50, "POST /api/orders")), + JourneyRun(search, None, report=_report(100, None))] + # together: breaks at 60 users in total, at checkout's order step. + # checkout had only 60 * 1/4 = 15 of its own users there, but held 40 alone. + block = combined_block(_report(50, 60, "checkout › POST /api/orders"), runs) + assert block["mix"] == {"checkout": 25.0, "search": 75.0} + assert block["culprit_journey"] == "checkout" + assert block["culprit_step"] == {"index": 2, "name": "place order", + "route": "POST /api/orders"} + assert block["interference"] == {"journey": "checkout", "users_at_break": 15.0, + "held_alone": 40, "with": ["search"]} + # breaking with more of its own users than it held alone is not interference + calm = combined_block(_report(150, 200, "checkout › POST /api/orders"), runs) + assert calm["interference"] is None + held = combined_block(_report(100, None), runs) + assert held["culprit_journey"] is None and held["interference"] is None + + +def test_expect_splits_users_across_journeys_by_weight(): + from prescale_cli.commands.run import expect_block + result = { + "verdict": {"survives_users": 20, "onset_users": 30}, + "journeys": [ + {"name": "browse", "status": "ok", "weight": 60, "survives_users": 400, + "onset_users": None}, + {"name": "checkout", "status": "ok", "weight": 5, "survives_users": 20, + "onset_users": 30}, + {"name": "old", "status": "stale", "weight": 5}, + ], + "combined": {"survives_users": 400, "onset_users": None}, + } + exp = expect_block(result, 300) + assert exp["meets"] is False + assert [(j["name"], j["users"], j["meets"]) for j in exp["journeys"]] == [ + ("browse", 276.9, True), ("checkout", 23.1, False)] + assert exp["combined"] == {"survives_users": 400, "meets": True} + # fewer expected users: checkout's share (~7.7) fits under the 20 it held + assert expect_block(result, 100)["meets"] is True + # more users than were ever tested on a journey that never broke: unknown + assert expect_block(result, 6000)["journeys"][0]["meets"] is None + + +def test_expect_on_a_url_run(): + from prescale_cli.commands.run import expect_block + broke = {"verdict": {"survives_users": 90, "onset_users": 100}} + assert expect_block(broke, 50)["meets"] is True + assert expect_block(broke, 200)["meets"] is False + held = {"verdict": {"survives_users": 200, "onset_users": None}} + assert expect_block(held, 500)["meets"] is None + + +def test_run_combined_pass_and_expect_end_to_end(tmp_path, monkeypatch): + _fake_checks(monkeypatch) + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--json", "--no-save", + "-u", "5", "--expect", "4", + "--dir", _dir(tmp_path, SHOP, BROWSE)]) + assert res.exit_code == 0, res.output + result = json.loads(res.stdout) + assert result["combined"]["mix"] == {"browse": 92.3, "checkout": 7.7} + assert result["expect"]["meets"] is True + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--json", "--no-save", + "-u", "5", "--no-combined", + "--dir", _dir(tmp_path, SHOP, BROWSE)]) + assert "combined" not in json.loads(res.stdout) From 6ba30381bea3f2601d8b0b0197cc8cd559492476 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 15:39:52 +0000 Subject: [PATCH 06/11] fix(journeys): pin users to journeys by weight so shares are exact --- CHANGELOG.md | 4 +- README.md | 6 +-- cli/src/prescale_cli/commands/run.py | 15 ++++-- cli/src/prescale_cli/journeys/load.py | 40 ++++++++++------ cli/src/prescale_cli/journeys/replay.py | 39 +++++++++++----- cli/src/prescale_cli/loadtest.py | 6 ++- cli/src/prescale_cli/render.py | 6 ++- .../prescale_cli/schema/result.schema.json | 7 +-- cli/tests/test_journeys_load.py | 46 ++++++++++++------- 9 files changed, 111 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7e87fb..d1bc541 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,8 @@ `verdict.culprit_journey`). Stale journeys are skipped with the reason. `--journey` picks journeys; `--urls` tests URLs as before. - **Journeys together, and interference.** After ramping each journey alone, `prescale run` - ramps them all together, each user picking a journey by `weight`, and flags a journey that - breaks with fewer of its own users than it held alone: it competes with the others for + ramps them all together, the users split between journeys by `weight`, and flags a journey that + breaks with no more of its own users than it held alone: it competes with the others for something they share (a database pool, CPU, a cache). Saved runs carry a `combined` field. `--no-combined` skips it. - **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split diff --git a/README.md b/README.md index 2caccce..2bed172 100644 --- a/README.md +++ b/README.md @@ -210,14 +210,14 @@ When `prescale/journeys/` has journeys, **`prescale run` replays them at load**: A stale journey isn't load-tested (a failure with no load says nothing about capacity); it's listed with the reason. `--journey NAME` picks journeys, and `--urls` (or `--path`, `--from-sitemap`) tests URLs instead. -Then **all journeys run together**, each user picking a journey by its `weight` (defaults by type: browse 60, search 25, account 8, checkout 5, write 5, auth 2; edit them in the files). That catches **interference**: journeys competing for something they share. If a journey breaks with fewer of its own users than it held alone, the report says so: +Then **all journeys run together**, the users split between journeys by `weight` (each user stays on its journey, so a weight is that journey's share of the users in the app at once) (defaults by type: browse 60, search 25, account 8, checkout 5, write 5, auth 2; edit them in the files). That catches **interference**: journeys competing for something they share. If a journey breaks with fewer of its own users than it held alone, the report says so: ``` -Interference checkout broke with only ~15 of its own users, but held ~20 alone. +Interference checkout broke with 15 of its own users, but held 20 alone. It competes with quickbuy for something they share (a database pool, CPU, a cache). ``` -`--no-combined` skips this pass. `--expect 500` says whether you'd hold 500 users at peak: in a journey run, each journey must hold its share of them (by weight) on its own, and all journeys together must hold 500. +`--no-combined` skips this pass. `--expect 500` says whether you'd hold 500 users at peak: in a journey run, each journey must hold its share of them (by weight) on its own, and all journeys together must hold 500; a journey that wasn't load-tested makes the answer unknown. It reports, it doesn't change the exit code (use `--fail-under` to gate CI). - `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. - **Secrets never go in the file.** A literal password, token, API key or `Authorization` value is refused; use `{{env.NAME}}`. diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index 90bb7bc..0dbe69c 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -98,7 +98,8 @@ @click.option("--no-combined", "no_combined", is_flag=True, help="Skip ramping all journeys together (by weight) after each one alone.") @click.option("--expect", "expect", default=None, type=click.IntRange(min=1), metavar="N", - help="Users you expect at peak; the verdict says whether you'd hold them.") + help="Users you expect at peak; the report says whether you'd hold them. " + "It doesn't change the exit code (use --fail-under for a CI gate).") @click.option("--urls", "urls_only", is_flag=True, help="Test URLs, not journeys, even when prescale/journeys/ has some.") @click.option("--allow-writes", "allow_writes", multiple=True, metavar="HOST", @@ -317,12 +318,14 @@ def expect_block(result: dict, users: int) -> dict: together (if they were ramped together) must hold USERS. It meets the expectation only if every one of those does.""" v = result["verdict"] + untested = [j["name"] for j in (result.get("journeys") or []) if j["status"] != "ok"] journeys = [j for j in (result.get("journeys") or []) if j["status"] == "ok"] if not journeys: return {"users": users, "meets": _meets(v["survives_users"], v["onset_users"] is None, users), "survives_users": v["survives_users"], "journeys": None, "combined": None} - total = sum(j["weight"] for j in journeys) + # A journey that wasn't load-tested still takes its share of the users. + total = sum(j["weight"] for j in result["journeys"]) per = [] for j in journeys: share = users * j["weight"] / total @@ -336,9 +339,11 @@ def expect_block(result: dict, users: int) -> dict: combined = {"survives_users": c["survives_users"], "meets": _meets(c["survives_users"], c["onset_users"] is None, users)} checks.append(combined["meets"]) + if untested: + checks.append(None) # their share was never tested meets = False if False in checks else (None if None in checks else True) return {"users": users, "meets": meets, "survives_users": v["survives_users"], - "journeys": per, "combined": combined} + "journeys": per, "combined": combined, "untested": untested} def _journeys_to_run(directory: Path, only: tuple[str, ...], *, urls_only: bool, @@ -436,7 +441,7 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa live_mode = isinstance(progress, LiveRamp) try: with progress: - stages, _ = asyncio.run(ramp( + stages, together_warning = asyncio.run(ramp( [r.journey for r in ramped], url, levels=levels, stage_seconds=config["stage_seconds"], timeout=timeout, max_rps=config["max_rps"], warmup=config["warmup"], @@ -448,6 +453,8 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa together = analyze(stages, latency_wall=latency_wall, error_threshold=error_threshold, rate_capped=config["max_rps"] is not None) result["combined"] = combined_block(together, runs) + if together_warning and together_warning not in (result["warning"] or ""): + result["warning"] = " ".join(filter(None, [result["warning"], together_warning])) return worst.report, result, live_mode diff --git a/cli/src/prescale_cli/journeys/load.py b/cli/src/prescale_cli/journeys/load.py index 139c064..9cd24a8 100644 --- a/cli/src/prescale_cli/journeys/load.py +++ b/cli/src/prescale_cli/journeys/load.py @@ -6,9 +6,9 @@ (how many users it survives, the step that breaks first, why) doesn't depend on any traffic mix. The run's overall verdict is the first journey to break. -The combined pass then ramps all of them together, each user picking a journey -by `weight`, to catch interference: journeys competing for something they share -(a database pool, CPU, a cache), which no journey shows on its own. +The combined pass then ramps all of them together, the users split between +journeys by `weight`, to catch interference: journeys competing for something +they share (a database pool, CPU, a cache), which no journey shows on its own. """ from __future__ import annotations @@ -18,7 +18,13 @@ import httpx from prescale_cli.journeys.model import Journey -from prescale_cli.journeys.replay import SEP, JourneyCheck, client_options, journey_vu +from prescale_cli.journeys.replay import ( + SEP, + JourneyCheck, + apportion, + client_options, + journey_vu, +) from prescale_cli.loadtest import RunReport, StageResult, run_loadtest @@ -132,11 +138,14 @@ def _step_stats(index: int, step, stage: StageResult | None) -> dict: def combined_block(report: RunReport, runs: list[JourneyRun]) -> dict: """The Result's `combined` field: the journeys ramped together by weight. - Users here are the total across journeys; each journey gets its weight's share. - If the journey that breaks first does so with fewer of its own users than it - held alone, something it shares with the others is the limit (interference).""" + Users here are the total across journeys, split between them by `apportion`, + so the number on each journey at any level is exact. If the journey that + breaks first does so with no more of its own users than it surely held alone + (the low end of its band), something it shares with the others is the limit: + interference.""" ramped = [r for r in runs if r.report is not None] - total = sum(r.journey.weight for r in ramped) + weights = [r.journey.weight for r in ramped] + total = sum(weights) share = {r.journey.name: r.journey.weight / total for r in ramped} block: dict = { "journeys": [r.journey.name for r in ramped], @@ -163,14 +172,17 @@ def combined_block(report: RunReport, runs: list[JourneyRun]) -> dict: if index is not None: block["culprit_step"] = {"index": index, "name": run.journey.steps[index - 1].name, "route": label} - users_of_it = report.onset_users * share[name] - alone = (run.report.survives_users if run.report.onset_users is not None - else run.report.max_tested) - if users_of_it < alone: + users_of_it = apportion(report.onset_users, weights)[ramped.index(run)] + alone = run.report + if alone.onset_users is None: + held = alone.max_tested + else: + held = alone.survives_low if alone.survives_low is not None else alone.survives_users + if 0 < users_of_it <= held: block["interference"] = { "journey": name, - "users_at_break": round(users_of_it, 1), - "held_alone": alone, + "users_at_break": users_of_it, + "held_alone": held, "with": [r.journey.name for r in ramped if r.journey.name != name], } return block diff --git a/cli/src/prescale_cli/journeys/replay.py b/cli/src/prescale_cli/journeys/replay.py index a79e9b3..633f226 100644 --- a/cli/src/prescale_cli/journeys/replay.py +++ b/cli/src/prescale_cli/journeys/replay.py @@ -18,7 +18,6 @@ import asyncio import os -import random import time from dataclasses import dataclass, field from http.cookiejar import CookieJar, DefaultCookiePolicy @@ -121,28 +120,44 @@ async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, return result +def apportion(users: int, weights: list[float]) -> list[int]: + """Split USERS into whole numbers proportional to WEIGHTS (largest remainder), + so the counts always add up to USERS.""" + total = sum(weights) + exact = [users * w / total for w in weights] + counts = [int(x) for x in exact] + by_remainder = sorted(range(len(weights)), key=lambda i: exact[i] - counts[i], reverse=True) + for i in by_remainder[:users - sum(counts)]: + counts[i] += 1 + return counts + + def journey_vu(journeys: Journey | list[Journey], base_url: str, - env: dict[str, str] | None = None, think: float = 0.0, - rng: random.Random | None = None): + env: dict[str, str] | None = None, think: float = 0.0): """A load-test virtual user: walk a journey's steps, pausing for each step's think time, and start again with a fresh identity until the stage deadline. A failed step ends that walk (after its pause), like a real user giving up. - Given several JOURNEYS, each walk picks one by `weight`, so they share the - app the way real traffic does; steps are then labelled `journey › step`. - Each step is recorded under its label, so the report can name the step that - broke. THINK is the pause for steps that don't set their own (`--think-time`, - or a launch profile's).""" + Given several JOURNEYS, the stage's users are split between them by `weight` + (see `apportion`) and each user stays on its journey for the whole stage, so + a journey's weight is its share of the users in the app at once, whatever + its walks take. Steps are then labelled `journey › step`. Each step is + recorded under its label, so the report can name the step that broke. THINK + is the pause for steps that don't set their own (`--think-time`, or a launch + profile's).""" env = dict(os.environ) if env is None else env mix = [journeys] if isinstance(journeys, Journey) else list(journeys) weights = [j.weight for j in mix] - rng = rng or random.Random() - async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None) -> None: + async def vu(client: httpx.AsyncClient, deadline: float, sink, gate=None, + index: int = 0, users: int = 1) -> None: loop = asyncio.get_running_loop() + journey = mix[0] + if len(mix) > 1: + seats = [j for j, n in zip(mix, apportion(users, weights)) for _ in range(n)] + journey = seats[index] + prefix = f"{journey.name}{SEP}" if len(mix) > 1 else "" while loop.time() < deadline: - journey = mix[0] if len(mix) == 1 else rng.choices(mix, weights)[0] - prefix = f"{journey.name}{SEP}" if len(mix) > 1 else "" variables: dict[str, Any] = fresh_uniques() jar = httpx.Cookies() for step in journey.steps: diff --git a/cli/src/prescale_cli/loadtest.py b/cli/src/prescale_cli/loadtest.py index 131d9f7..adb4305 100644 --- a/cli/src/prescale_cli/loadtest.py +++ b/cli/src/prescale_cli/loadtest.py @@ -394,7 +394,8 @@ async def _run_stage(client: httpx.AsyncClient, targets: list[str], method: str, think_time: float = 0.0, vu=None) -> StageResult: """Hold USERS virtual users for DURATION. By default each fires requests at TARGETS; VU, when given, is the virtual user itself: an async callable - (client, deadline, sink, gate) that records its own requests (journeys).""" + (client, deadline, sink, gate, index, users) that records its own requests + (journeys). INDEX is this user's place among the stage's USERS.""" sink = _Sink(labels=vu is None) cycle = itertools.cycle(targets) # round-robin spreads load evenly across routes loop = asyncio.get_running_loop() @@ -405,7 +406,8 @@ async def _run_stage(client: httpx.AsyncClient, targets: list[str], method: str, watcher = asyncio.create_task(_watch_loop_lag(stop, lags)) try: if vu is not None: - await asyncio.gather(*(vu(client, deadline, sink, gate) for _ in range(users))) + await asyncio.gather(*(vu(client, deadline, sink, gate, i, users) + for i in range(users))) else: await asyncio.gather( *(_worker(client, method, deadline, sink, lambda: next(cycle), gate, think_time) diff --git a/cli/src/prescale_cli/render.py b/cli/src/prescale_cli/render.py index 887d671..fe5e821 100644 --- a/cli/src/prescale_cli/render.py +++ b/cli/src/prescale_cli/render.py @@ -159,6 +159,8 @@ def _expect_lines(exp: dict) -> list[str]: if c: mark = {True: "✅", False: "🛑", None: "❔"}[c["meets"]] lines.append(f" {mark} all together: holds ~{c['survives_users']}") + for name in exp.get("untested") or []: + lines.append(f" ❔ {escape(name)}: not load-tested (stale or unreachable)") return lines @@ -212,8 +214,8 @@ def _render_combined(c: dict) -> None: if inter: others = ", ".join(escape(n) for n in inter["with"]) lines.append( - f"[yellow]Interference[/yellow] {escape(inter['journey'])} broke with only " - f"~{inter['users_at_break']:g} of its own users, but held ~{inter['held_alone']} " + f"[yellow]Interference[/yellow] {escape(inter['journey'])} broke with " + f"{inter['users_at_break']} of its own users, but held {inter['held_alone']} " f"alone. It competes with {others} for something they share (a database pool, " "CPU, a cache).") console.print(Panel("\n".join(lines), title="All journeys together", border_style="cyan")) diff --git a/cli/src/prescale_cli/schema/result.schema.json b/cli/src/prescale_cli/schema/result.schema.json index f8a4072..7751c93 100644 --- a/cli/src/prescale_cli/schema/result.schema.json +++ b/cli/src/prescale_cli/schema/result.schema.json @@ -103,7 +103,7 @@ "description": "Journey runs with two or more journeys that replay: all of them ramped together, each user picking a journey by weight. Users are the total across journeys.", "properties": { "journeys": { "type": "array", "items": { "type": "string" } }, - "mix": { "type": "object", "additionalProperties": { "type": "number" }, "description": "Percent of users per journey." }, + "mix": { "type": "object", "additionalProperties": { "type": "number" }, "description": "Percent of the concurrent users on each journey (by weight; each user stays on one journey)." }, "survives_users": { "type": "integer" }, "max_tested": { "type": "integer" }, "onset_users": { "type": ["integer", "null"] }, @@ -117,7 +117,7 @@ "description": "Set when the journey that breaks first does so with fewer of its own users than it held alone: it competes with the others for something they share.", "properties": { "journey": { "type": "string" }, - "users_at_break": { "type": "number" }, + "users_at_break": { "type": "integer", "description": "Its own users (exact) at the level where the combined run broke." }, "held_alone": { "type": "integer" }, "with": { "type": "array", "items": { "type": "string" } } } @@ -132,7 +132,8 @@ "meets": { "type": ["boolean", "null"], "description": "null: held every level tested, but fewer users than expected were tested." }, "survives_users": { "type": "integer" }, "journeys": { "type": ["array", "null"] }, - "combined": { "type": ["object", "null"] } + "combined": { "type": ["object", "null"] }, + "untested": { "type": "array", "items": { "type": "string" }, "description": "Journeys that weren't load-tested; their share of N is unknown." } } }, "investigation": { diff --git a/cli/tests/test_journeys_load.py b/cli/tests/test_journeys_load.py index aac61e2..fecb4fb 100644 --- a/cli/tests/test_journeys_load.py +++ b/cli/tests/test_journeys_load.py @@ -264,25 +264,35 @@ async def spy(journey, url, **kwargs): # --- the combined pass and --expect ------------------------------------------------ -def test_mixed_users_pick_journeys_by_weight_and_label_steps_by_journey(): - import random +def test_apportion_adds_up_and_follows_weights(): + from prescale_cli.journeys.replay import apportion + assert apportion(10, [9, 1]) == [9, 1] + assert apportion(30, [1, 1]) == [15, 15] + assert apportion(3, [60, 25, 8, 5, 2]) == [2, 1, 0, 0, 0] + assert sum(apportion(97, [60, 25, 8, 5, 2])) == 97 + +def test_mixed_users_stay_on_their_journey_whatever_its_walk_takes(): from prescale_cli.journeys.replay import journey_vu from prescale_cli.loadtest import _Sink - heavy = parse({**BROWSE, "name": "heavy", "weight": 9}) - light = parse({**BROWSE, "name": "light", "weight": 1, - "steps": [{"request": {"path": "/light"}}]}) - vu = journey_vu([heavy, light], "http://shop.test", env={}, rng=random.Random(7)) - sink = _Sink(labels=False) + quick = parse({**BROWSE, "name": "quick", "weight": 1}) + slow = parse({**BROWSE, "name": "slow", "weight": 1, + "steps": [{"request": {"path": "/slow"}, "think": "1s"}]}) + vu = journey_vu([quick, slow], "http://shop.test", env={}) + sinks = [_Sink(labels=False) for _ in range(10)] async def go(): - async with httpx.AsyncClient(transport=httpx.MockTransport(lambda r: httpx.Response(200))) as c: - await vu(c, asyncio.get_running_loop().time() + 0.2, sink) + transport = httpx.MockTransport(lambda r: httpx.Response(200)) + async with httpx.AsyncClient(transport=transport) as c: + deadline = asyncio.get_running_loop().time() + 0.3 + await asyncio.gather(*(vu(c, deadline, sinks[i], None, i, 10) for i in range(10))) asyncio.run(go()) - counts = {label: stat.total for label, stat in sink.routes.items()} - assert set(counts) == {"heavy › GET /", "light › GET /light"} - assert counts["heavy › GET /"] > 4 * counts["light › GET /light"] + # each user only ever walked one journey; half the users are on each + walked = [set(s.routes) for s in sinks] + assert all(len(w) == 1 for w in walked) + assert sum(w == {"quick › GET /"} for w in walked) == 5 + assert sum(w == {"slow › GET /slow"} for w in walked) == 5 def _report(survives, onset, culprit=None, max_tested=100): @@ -329,13 +339,17 @@ def test_expect_splits_users_across_journeys_by_weight(): } exp = expect_block(result, 300) assert exp["meets"] is False + # the stale journey still takes its share of the users (weights total 70) assert [(j["name"], j["users"], j["meets"]) for j in exp["journeys"]] == [ - ("browse", 276.9, True), ("checkout", 23.1, False)] + ("browse", 257.1, True), ("checkout", 21.4, False)] assert exp["combined"] == {"survives_users": 400, "meets": True} - # fewer expected users: checkout's share (~7.7) fits under the 20 it held - assert expect_block(result, 100)["meets"] is True + assert exp["untested"] == ["old"] + # every tested share fits, but "old" was never tested: unknown, not a yes + assert expect_block(result, 100)["meets"] is None + fresh = {**result, "journeys": result["journeys"][:2]} + assert expect_block(fresh, 100)["meets"] is True # more users than were ever tested on a journey that never broke: unknown - assert expect_block(result, 6000)["journeys"][0]["meets"] is None + assert expect_block(fresh, 6000)["journeys"][0]["meets"] is None def test_expect_on_a_url_run(): From b2a186406625d3ab537ac76f20444a42e694de7d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:04:45 +0000 Subject: [PATCH 07/11] feat(journeys): journeys in the HTML report, compare and MCP summaries --- CHANGELOG.md | 4 + README.md | 2 + cli/src/prescale_cli/commands/compare.py | 16 +++ cli/src/prescale_cli/compare.py | 93 +++++++++++++++--- cli/src/prescale_cli/mcp_tools.py | 30 ++++++ cli/src/prescale_cli/report.py | 80 +++++++++++++++ cli/tests/test_journeys_report.py | 120 +++++++++++++++++++++++ 7 files changed, 330 insertions(+), 15 deletions(-) create mode 100644 cli/tests/test_journeys_report.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d1bc541..f076227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ breaks with no more of its own users than it held alone: it competes with the others for something they share (a database pool, CPU, a cache). Saved runs carry a `combined` field. `--no-combined` skips it. +- **Journeys everywhere.** The `--html` report shows each journey, the combined run and + `--expect`; the MCP tools' run summaries list each journey's verdict; and `prescale compare` + diffs each journey by name (terminal, `--json`, `--markdown`). A confident drop in any journey + counts as a regression for `--fail-on-regression`, even when it isn't the weakest journey. - **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split across journeys by weight: each must hold its share alone, and all together must hold N. diff --git a/README.md b/README.md index 2bed172..72fbffb 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,8 @@ Interference checkout broke with 15 of its own users, but held 20 alone. It competes with quickbuy for something they share (a database pool, CPU, a cache). ``` +`prescale show`, `--html` reports and the MCP tools show journeys too, and `prescale compare` diffs each journey: a confident drop in any journey is a regression (so `--fail-on-regression` catches it), even when it isn't the weakest one. A URL run and a journey run measure different things, so comparing them is reported as not comparable. + `--no-combined` skips this pass. `--expect 500` says whether you'd hold 500 users at peak: in a journey run, each journey must hold its share of them (by weight) on its own, and all journeys together must hold 500; a journey that wasn't load-tested makes the answer unknown. It reports, it doesn't change the exit code (use `--fail-under` to gate CI). - `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. diff --git a/cli/src/prescale_cli/commands/compare.py b/cli/src/prescale_cli/commands/compare.py index a6cb951..a569fca 100644 --- a/cli/src/prescale_cli/commands/compare.py +++ b/cli/src/prescale_cli/commands/compare.py @@ -6,6 +6,7 @@ import click from rich.console import Console +from rich.markup import escape from rich.panel import Panel from rich.table import Table @@ -97,3 +98,18 @@ def _render(cmp: dict) -> None: note = "" if (cmp["confident"] or not cmp["regressed"]) else \ " (confidence bands overlap — could be noise)" console.print(Panel(cmp["summary"] + note, border_style=color)) + if cmp.get("journeys"): + jt = Table(show_header=True, header_style="bold magenta", title="Per journey") + for col, justify in (("Journey", "left"), ("Old", "right"), ("New", "right"), + ("Change", "right"), ("Breaks at", "left")): + jt.add_column(col, justify=justify) + for j in cmp["journeys"]: + if j["status"] != "ok": + jt.add_row(escape(j["name"]), "", "", f"[dim]{escape(j['status'])}[/dim]", "") + continue + tone = "red" if j["regressed"] else ("green" if j.get("improved") else "") + change = f"{j['pct']:+.0%}" + jt.add_row(escape(j["name"]), f"~{j['old']}", f"~{j['new']}", + f"[{tone}]{change}[/{tone}]" if tone else change, + escape(j.get("breaks_at") or "—")) + console.print(jt) diff --git a/cli/src/prescale_cli/compare.py b/cli/src/prescale_cli/compare.py index dea8b2e..9a8d9e8 100644 --- a/cli/src/prescale_cli/compare.py +++ b/cli/src/prescale_cli/compare.py @@ -40,6 +40,37 @@ def compare_results(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHO if o["mode"] != n["mode"]: # URL capacity and journey capacity measure different things: never a regression. return _not_comparable(o, n) + d = _diff(o, n, threshold) + cmp = { + "old": o, + "new": n, + "survives_delta": n["survives_users"] - o["survives_users"], + "survives_pct": round(d["pct"], 3), + "peak_rps_pct": (round((n["peak_rps"] - o["peak_rps"]) / o["peak_rps"], 3) + if o["peak_rps"] else 0.0), + "regressed": d["regressed"], + "improved": d["improved"], + "inconclusive": d["inconclusive"], + "confident": d["confident"], + "summary": (_summary(o, n, d["pct"], d["regressed"], d["improved"], d["inconclusive"]) + + _journey_note(o, n)), + } + if o["mode"] == "journeys": + cmp["journeys"] = compare_journeys(old, new, threshold=threshold) + worse = [j for j in cmp["journeys"] if j.get("regressed")] + if worse: + # A drop in any one journey is a regression, even if it isn't the weakest. + cmp["regressed"] = True + cmp["confident"] = cmp["confident"] or any(j["confident"] for j in worse) + if not d["regressed"]: + cmp["summary"] = "Capacity regressed: " + "; ".join( + f"{j['name']} {j['old']} → {j['new']} users ({j['pct']:+.0%})" + for j in worse) + "." + return cmp + + +def _diff(o: dict, n: dict, threshold: float) -> dict: + """The capacity change between two sets of facts (a run, or one journey).""" o_surv, n_surv = o["survives_users"], n["survives_users"] pct = (n_surv - o_surv) / o_surv if o_surv else 0.0 inconclusive = (n["onset_users"] is None and n["max_tested"] is not None @@ -51,19 +82,41 @@ def compare_results(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHO confident = n["survives_high"] < o["survives_low"] elif improved and o["survives_high"] is not None and n["survives_low"] is not None: confident = n["survives_low"] > o["survives_high"] - return { - "old": o, - "new": n, - "survives_delta": n_surv - o_surv, - "survives_pct": round(pct, 3), - "peak_rps_pct": (round((n["peak_rps"] - o["peak_rps"]) / o["peak_rps"], 3) - if o["peak_rps"] else 0.0), - "regressed": regressed, - "improved": improved, - "inconclusive": inconclusive, - "confident": confident, - "summary": _summary(o, n, pct, regressed, improved, inconclusive) + _journey_note(o, n), - } + return {"pct": pct, "regressed": regressed, "improved": improved, + "inconclusive": inconclusive, "confident": confident} + + +def _journey_facts(j: dict) -> dict: + conf = j.get("confidence") or {} + return {"survives_users": j["survives_users"], "survives_low": conf.get("survives_low"), + "survives_high": conf.get("survives_high"), "onset_users": j["onset_users"], + "max_tested": j.get("max_tested")} + + +def compare_journeys(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHOLD) -> list: + """Per-journey capacity change, matched by name. A journey that is new, gone, + or not load-tested on either side is listed with its status, never as a drop.""" + before = {j["name"]: j for j in old.get("journeys") or []} + after = {j["name"]: j for j in new.get("journeys") or []} + rows = [] + for name in [*before, *(n for n in after if n not in before)]: + o, n = before.get(name), after.get(name) + if o is None or n is None: + rows.append({"name": name, "status": "new" if o is None else "removed", + "regressed": False, "confident": False}) + continue + if o["status"] != "ok" or n["status"] != "ok": + rows.append({"name": name, "status": f"{o['status']} → {n['status']}", + "regressed": False, "confident": False}) + continue + d = _diff(_journey_facts(o), _journey_facts(n), threshold) + step = (n.get("culprit_step") or {}).get("name") + rows.append({"name": name, "status": "ok", "old": o["survives_users"], + "new": n["survives_users"], "pct": round(d["pct"], 3), + "regressed": d["regressed"], "improved": d["improved"], + "inconclusive": d["inconclusive"], "confident": d["confident"], + "breaks_at": step}) + return rows def _not_comparable(o: dict, n: dict) -> dict: @@ -117,7 +170,7 @@ def to_markdown(cmp: dict) -> str: emoji = "🟢" else: emoji = "⚪" - return "\n".join([ + lines = [ f"### {emoji} PreScale — {cmp['summary']}", "", "| | survives | peak req/s | commit |", @@ -132,4 +185,14 @@ def to_markdown(cmp: dict) -> str: max(o["survives_users"], n["survives_users"], 1) * 1.1, ), "```", - ]) + ] + if cmp.get("journeys"): + lines += ["", "| journey | old | new | change | breaks at |", "|---|---:|---:|---:|---|"] + for j in cmp["journeys"]: + if j["status"] != "ok": + lines.append(f"| {j['name']} | | | {j['status']} | |") + continue + mark = " 🔴" if j["regressed"] else (" 🟢" if j.get("improved") else "") + lines.append(f"| {j['name']} | ~{j['old']} | ~{j['new']} | {j['pct']:+.0%}{mark} | " + f"{j.get('breaks_at') or '—'} |") + return "\n".join(lines) diff --git a/cli/src/prescale_cli/mcp_tools.py b/cli/src/prescale_cli/mcp_tools.py index 1278e87..da3e646 100644 --- a/cli/src/prescale_cli/mcp_tools.py +++ b/cli/src/prescale_cli/mcp_tools.py @@ -77,10 +77,40 @@ def summarize_result(result: dict) -> dict: "peak_rps": v["peak_rps"], "saturated": v["saturated"], "warning": result.get("warning"), + **_journey_summary(result), "hint": "Call get_run(id) for the full per-level / per-route breakdown.", } +def _journey_summary(result: dict) -> dict: + """Journey runs: one line per journey, plus interference and --expect, kept short.""" + journeys = result.get("journeys") + if not journeys: + return {} + out: dict = {"culprit_journey": result["verdict"].get("culprit_journey"), "journeys": []} + for j in journeys: + if j["status"] != "ok": + out["journeys"].append({"name": j["name"], "status": j["status"], + "reason": j.get("reason")}) + continue + step = j.get("culprit_step") + out["journeys"].append({ + "name": j["name"], "status": "ok", + "survives_users": j["survives_users"], "held_all": j["onset_users"] is None, + "breaks_at_step": (f"{step['index']}. {step['name']} ({step['route']})" + if step else None), + "bottleneck": j.get("bottleneck"), + }) + combined = result.get("combined") + if combined: + out["together"] = {"survives_users": combined["survives_users"], + "held_all": combined["onset_users"] is None, + "interference": combined.get("interference")} + if result.get("expect"): + out["expect"] = {k: result["expect"].get(k) for k in ("users", "meets", "untested")} + return out + + def _verdict_line(v: dict, conf: dict) -> str: if v["onset_users"] is None: return f"Held up through {v['max_tested']} concurrent users (the most tested)." diff --git a/cli/src/prescale_cli/report.py b/cli/src/prescale_cli/report.py index 5de319f..07720b6 100644 --- a/cli/src/prescale_cli/report.py +++ b/cli/src/prescale_cli/report.py @@ -80,6 +80,10 @@ .cause .lbl{font-size:11px;font-weight:500;text-transform:uppercase;letter-spacing:.5px; color:var(--amber);margin-bottom:6px;} .cause p{font-size:13.5px;color:var(--muted);} +.cause ul{margin:4px 0 0 18px;font-size:13.5px;color:var(--muted);} +.note{font-size:12.5px;color:var(--muted);margin:-4px 0 8px;} +td.t,th.t{text-align:left;} +td.n{white-space:nowrap;} .cause p+p{margin-top:8px;} .cause b{color:var(--ink);} footer{margin-top:44px;padding-top:20px;border-top:1px solid var(--hair);display:flex; @@ -262,6 +266,81 @@ def _cause(result: dict) -> str: return f'
Likely cause
{"".join(paras)}
' +def _journeys_html(result: dict) -> str: + """Journey runs: each journey's verdict, the combined pass, and --expect.""" + journeys = result.get("journeys") + if not journeys: + return "" + first = result["verdict"].get("culprit_journey") + rows = [] + for j in journeys: + if j["status"] != "ok": + label = "stale" if j["status"] == "stale" else "couldn't run" + rows.append(f"{_esc(j['name'])}{label}" + f"—" + f"{_esc(j.get('reason') or '')}") + continue + if j["onset_users"] is None: + survives = f"{j['max_tested']}+ (held)" + else: + survives = f"~{j['survives_users']}{_band(j)}" + step = j.get("culprit_step") + where = (f"{step['index']}. {_esc(step['name'])} {_esc(step['route'])}" + if step else "—") + cls = ' class="onset"' if j["name"] == first and j["onset_users"] is not None else "" + rows.append(f"{_esc(j['name'])}{survives}" + f"{where}" + f"{_esc(j.get('bottleneck') or '')}") + parts = [ + '

Journeys

' + '

Each journey ramped on its own.

' + '
' + "" + + "".join(rows) + "
JourneySurvivesBreaks at stepWhy
" + ] + c = result.get("combined") + if c: + mix = ", ".join(f"{_esc(n)} {pct:g}%" for n, pct in c["mix"].items()) + paras = [f"

All journeys together ({mix}): " + + (f"held through {c['max_tested']} users in total.

" + if c["onset_users"] is None else + f"survives ~{c['survives_users']} users in total" + + (f"; breaks first at {_esc(c['culprit_journey'])}." + if c.get("culprit_journey") else ".") + "

")] + inter = c.get("interference") + if inter: + paras.append( + f"

Interference: {_esc(inter['journey'])} broke with " + f"{inter['users_at_break']} of its own users, but held {inter['held_alone']} " + f"alone. It competes with {_esc(', '.join(inter['with']))} for something " + "they share (a database pool, CPU, a cache).

") + parts.append('
Together
' + + "".join(paras) + "
") + parts.append(_expect_html(result.get("expect"))) + parts.append("
") + return "".join(parts) + + +def _expect_html(exp: dict | None) -> str: + if not exp: + return "" + answer = {True: "yes", False: "no", None: "unknown"}[exp["meets"]] + items = [] + for j in exp.get("journeys") or []: + mark = {True: "holds", False: "doesn't hold", None: "unknown"}[j["meets"]] + items.append(f"
  • {_esc(j['name'])}: ~{j['users']:g} users, holds " + f"~{j['survives_users']} alone ({mark})
    • ") + combined = exp.get("combined") + if combined: + items.append(f"
  • all together: holds ~{combined['survives_users']}
    • ") + for name in exp.get("untested") or []: + items.append(f"
  • {_esc(name)}: not load-tested (stale or unreachable)
    • ") + if not exp.get("journeys"): + items.append(f"
  • survives ~{exp['survives_users']}
    • ") + return (f'
    Expected ~{exp["users"]} users: {answer}' + f"
      {''.join(items)}
    ") + + def render_html(result: dict) -> str: target = result["target"] config = result["config"] @@ -307,6 +386,7 @@ def render_html(result: dict) -> str: {_cause(result)} {_route_table(result)} + {_journeys_html(result)}
    Generated by prescale {result["tool_version"]} {meta}
    diff --git a/cli/tests/test_journeys_report.py b/cli/tests/test_journeys_report.py new file mode 100644 index 0000000..031da19 --- /dev/null +++ b/cli/tests/test_journeys_report.py @@ -0,0 +1,120 @@ +"""Journey results everywhere: compare, the HTML report, and the MCP summary.""" + +import copy + +from prescale_cli.compare import compare_results, to_markdown +from prescale_cli.mcp_tools import summarize_result +from prescale_cli.report import render_html + + +def _journey(name, survives, onset, low=None, high=None, step=None, status="ok"): + if status != "ok": + return {"name": name, "type": "browse", "weight": 5, "status": status, + "reason": "step 1 (home): returned 404, expected 200"} + return {"name": name, "type": "checkout", "weight": 5, "status": "ok", "reason": None, + "survives_users": survives, "max_tested": 200, "onset_users": onset, + "onset_reason": "errors" if onset else None, + "confidence": {"survives_low": low if low is not None else survives, + "survives_high": high if high is not None else survives, + "stable": True}, + "culprit_step": ({"index": 3, "name": step, "route": "POST /api/orders"} + if step else None), + "bottleneck": "5xx under load" if onset else None} + + +def _run(journeys, survives=20, culprit="checkout"): + return { + "id": "r1", "created_at": "2026-11-02T10:00:00Z", "tool_version": "1.1.3", + "target": {"url": "http://shop.test", "host": "shop.test", "routes": ["GET /"]}, + "config": {"method": "GET", "max_users": 200, "stage_seconds": 2, + "latency_wall_s": 2.0, "error_threshold": 0.02, "max_rps": None, + "mode": "journeys"}, + "verdict": {"survives_users": survives, "max_tested": 200, "onset_users": survives + 10, + "onset_reason": "errors", "culprit_route": "POST /api/orders", + "culprit_journey": culprit, "bottleneck": "5xx under load", + "saturated": False, "saturation_users": None, "peak_rps": 300.0, + "marginal": False, + "confidence": {"survives_low": survives, "survives_high": survives, + "stable": True}}, + "stages": [{"users": 1, "rps": 3.0, "p50_ms": 20, "p95_ms": 30, "p99_ms": 40, + "error_rate": 0.0, "errors": 0, "total": 6, "routes": {}}], + "environment": {"git_commit": "abc1234"}, + "journeys": journeys, + } + + +def test_compare_flags_a_drop_in_any_journey(): + old = _run([_journey("browse", 200, None), _journey("checkout", 40, 50, step="place order")], + survives=40) + new = _run([_journey("browse", 200, None), _journey("checkout", 20, 30, step="place order")], + survives=20) + cmp = compare_results(old, new) + rows = {j["name"]: j for j in cmp["journeys"]} + assert rows["checkout"]["regressed"] and rows["checkout"]["confident"] + assert rows["checkout"]["pct"] == -0.5 and rows["checkout"]["breaks_at"] == "place order" + assert not rows["browse"]["regressed"] + assert cmp["regressed"] + assert "| checkout | ~40 | ~20 | -50% 🔴 | place order |" in to_markdown(cmp) + + +def test_a_drop_in_a_journey_that_isnt_the_weakest_is_still_a_regression(): + # search is the weakest both times; checkout drops but stays above it + old = _run([_journey("search", 10, 20), _journey("checkout", 100, 120)], survives=10, + culprit="search") + new = _run([_journey("search", 10, 20), _journey("checkout", 50, 60)], survives=10, + culprit="search") + cmp = compare_results(old, new) + assert cmp["survives_delta"] == 0 + assert cmp["regressed"] and cmp["confident"] + assert cmp["summary"] == "Capacity regressed: checkout 100 → 50 users (-50%)." + + +def test_new_removed_and_stale_journeys_are_listed_not_counted(): + old = _run([_journey("browse", 200, None), _journey("gone", 30, 40)]) + new = _run([_journey("browse", 200, None, status="stale"), _journey("fresh", 30, 40)]) + rows = {j["name"]: j["status"] for j in compare_results(old, new)["journeys"]} + assert rows == {"browse": "ok → stale", "gone": "removed", "fresh": "new"} + assert not any(j["regressed"] for j in compare_results(old, new)["journeys"]) + + +def test_html_report_shows_journeys_combined_and_expect(): + run = _run([_journey("checkout", 20, 30, low=20, high=30, step="place order"), + _journey("old", 0, None, status="stale")]) + run["combined"] = {"mix": {"checkout": 50.0, "quickbuy": 50.0}, "survives_users": 20, + "max_tested": 200, "onset_users": 30, "culprit_journey": "checkout", + "interference": {"journey": "checkout", "users_at_break": 15, + "held_alone": 20, "with": ["quickbuy"]}} + run["expect"] = {"users": 60, "meets": False, "survives_users": 20, + "journeys": [{"name": "checkout", "users": 30.0, "survives_users": 20, + "meets": False}], + "combined": {"survives_users": 20, "meets": False}, "untested": ["old"]} + html = render_html(run) + assert "

    Journeys

    " in html + assert "~20 (20–30)" in html and "3. place order" in html + assert "checkout broke with 15 of its own users, but held 20 alone" in html + assert "Expected ~60 users: no" in html and "old: not load-tested" in html + + +def test_html_escapes_journey_names_and_reasons(): + j = _journey("x", 20, 30, step="") + html = render_html(_run([j])) + assert "" not in html + + +def test_url_runs_render_and_compare_as_before(): + run = _run([]) + del run["journeys"] + run["config"].pop("mode") + assert "

    Journeys

    " not in render_html(run) + assert "journeys" not in compare_results(run, copy.deepcopy(run)) + assert "journeys" not in summarize_result(run) + + +def test_mcp_summary_lists_journeys_compactly(): + run = _run([_journey("checkout", 20, 30, step="place order"), + _journey("old", 0, None, status="stale")]) + s = summarize_result(run) + assert s["culprit_journey"] == "checkout" + assert s["journeys"][0]["breaks_at_step"] == "3. place order (POST /api/orders)" + assert s["journeys"][1] == {"name": "old", "status": "stale", + "reason": "step 1 (home): returned 404, expected 200"} From d5ac13dae3cd14b02d10fca48173b9b704de6f1f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:09:17 +0000 Subject: [PATCH 08/11] fix(journeys): honest compare labels: confidence from drops, inconclusive rows, escaped cells --- CHANGELOG.md | 4 +-- README.md | 2 +- cli/src/prescale_cli/commands/compare.py | 12 +++++-- cli/src/prescale_cli/compare.py | 41 +++++++++++++++++++++--- cli/tests/test_journeys_report.py | 28 ++++++++++++++++ 5 files changed, 76 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f076227..90f9544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,8 +27,8 @@ `--no-combined` skips it. - **Journeys everywhere.** The `--html` report shows each journey, the combined run and `--expect`; the MCP tools' run summaries list each journey's verdict; and `prescale compare` - diffs each journey by name (terminal, `--json`, `--markdown`). A confident drop in any journey - counts as a regression for `--fail-on-regression`, even when it isn't the weakest journey. + diffs each journey by name (terminal, `--json`, `--markdown`). A drop past the threshold in any + journey counts as a regression for `--fail-on-regression`, even when it isn't the weakest journey. - **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split across journeys by weight: each must hold its share alone, and all together must hold N. diff --git a/README.md b/README.md index 72fbffb..19d7d27 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ Interference checkout broke with 15 of its own users, but held 20 alone. It competes with quickbuy for something they share (a database pool, CPU, a cache). ``` -`prescale show`, `--html` reports and the MCP tools show journeys too, and `prescale compare` diffs each journey: a confident drop in any journey is a regression (so `--fail-on-regression` catches it), even when it isn't the weakest one. A URL run and a journey run measure different things, so comparing them is reported as not comparable. +`prescale show`, `--html` reports and the MCP tools show journeys too, and `prescale compare` diffs each journey: a drop past the threshold in any journey is a regression (so `--fail-on-regression` catches it), even when it isn't the weakest one; it's marked confident when the confidence bands don't overlap. A journey that held everything it was tested to, below the old level, is inconclusive, not a drop. A URL run and a journey run measure different things, so comparing them is reported as not comparable. `--no-combined` skips this pass. `--expect 500` says whether you'd hold 500 users at peak: in a journey run, each journey must hold its share of them (by weight) on its own, and all journeys together must hold 500; a journey that wasn't load-tested makes the answer unknown. It reports, it doesn't change the exit code (use `--fail-under` to gate CI). diff --git a/cli/src/prescale_cli/commands/compare.py b/cli/src/prescale_cli/commands/compare.py index a569fca..9b90454 100644 --- a/cli/src/prescale_cli/commands/compare.py +++ b/cli/src/prescale_cli/commands/compare.py @@ -11,7 +11,13 @@ from rich.table import Table from prescale_cli.bars import text_bar -from prescale_cli.compare import compare_results, resolve, to_markdown +from prescale_cli.compare import ( + compare_results, + journey_change, + journey_users, + resolve, + to_markdown, +) from prescale_cli.result import AmbiguousResultError, ResultNotFoundError console = Console() @@ -108,8 +114,8 @@ def _render(cmp: dict) -> None: jt.add_row(escape(j["name"]), "", "", f"[dim]{escape(j['status'])}[/dim]", "") continue tone = "red" if j["regressed"] else ("green" if j.get("improved") else "") - change = f"{j['pct']:+.0%}" - jt.add_row(escape(j["name"]), f"~{j['old']}", f"~{j['new']}", + change = journey_change(j) + jt.add_row(escape(j["name"]), journey_users(j, "old"), journey_users(j, "new"), f"[{tone}]{change}[/{tone}]" if tone else change, escape(j.get("breaks_at") or "—")) console.print(jt) diff --git a/cli/src/prescale_cli/compare.py b/cli/src/prescale_cli/compare.py index 9a8d9e8..6f9ae91 100644 --- a/cli/src/prescale_cli/compare.py +++ b/cli/src/prescale_cli/compare.py @@ -60,12 +60,19 @@ def compare_results(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESHO worse = [j for j in cmp["journeys"] if j.get("regressed")] if worse: # A drop in any one journey is a regression, even if it isn't the weakest. + # Confidence only comes from drops: a confident overall gain says nothing + # about whether a journey's drop is noise. cmp["regressed"] = True - cmp["confident"] = cmp["confident"] or any(j["confident"] for j in worse) + cmp["confident"] = ((d["confident"] and d["regressed"]) + or any(j["confident"] for j in worse)) if not d["regressed"]: + cmp["improved"] = False cmp["summary"] = "Capacity regressed: " + "; ".join( f"{j['name']} {j['old']} → {j['new']} users ({j['pct']:+.0%})" for j in worse) + "." + if d["improved"]: + cmp["summary"] += (f" (Overall: {o['survives_users']} → " + f"{n['survives_users']} users, {d['pct']:+.0%}.)") return cmp @@ -113,12 +120,35 @@ def compare_journeys(old: dict, new: dict, *, threshold: float = _DEFAULT_THRESH step = (n.get("culprit_step") or {}).get("name") rows.append({"name": name, "status": "ok", "old": o["survives_users"], "new": n["survives_users"], "pct": round(d["pct"], 3), + "old_held_all": o["onset_users"] is None, + "new_held_all": n["onset_users"] is None, "regressed": d["regressed"], "improved": d["improved"], "inconclusive": d["inconclusive"], "confident": d["confident"], "breaks_at": step}) return rows +def journey_users(row: dict, side: str) -> str: + """A journey's capacity in a compare row: "~20", or "200+" if it held every level.""" + return f"{row[side]}+" if row.get(f"{side}_held_all") else f"~{row[side]}" + + +def journey_change(row: dict, *, markers: bool = False) -> str: + """The change cell: the percentage, or "inconclusive" when the new run held + everything it tested but was never pushed as far as the old one.""" + if row.get("inconclusive"): + return "inconclusive" + mark = "" + if markers: + mark = " 🔴" if row["regressed"] else (" 🟢" if row.get("improved") else "") + return f"{row['pct']:+.0%}{mark}" + + +def _cell(text: str) -> str: + """Text safe inside a markdown table cell (journey and step names come from YAML).""" + return str(text).replace("|", "\\|").replace("\n", " ") + + def _not_comparable(o: dict, n: dict) -> dict: kinds = {"urls": "tested URLs", "journeys": "replayed journeys"} return { @@ -189,10 +219,11 @@ def to_markdown(cmp: dict) -> str: if cmp.get("journeys"): lines += ["", "| journey | old | new | change | breaks at |", "|---|---:|---:|---:|---|"] for j in cmp["journeys"]: + name = _cell(j["name"]) if j["status"] != "ok": - lines.append(f"| {j['name']} | | | {j['status']} | |") + lines.append(f"| {name} | | | {j['status']} | |") continue - mark = " 🔴" if j["regressed"] else (" 🟢" if j.get("improved") else "") - lines.append(f"| {j['name']} | ~{j['old']} | ~{j['new']} | {j['pct']:+.0%}{mark} | " - f"{j.get('breaks_at') or '—'} |") + step = _cell(j.get("breaks_at") or "—") + lines.append(f"| {name} | {journey_users(j, 'old')} | {journey_users(j, 'new')} | " + f"{journey_change(j, markers=True)} | {step} |") return "\n".join(lines) diff --git a/cli/tests/test_journeys_report.py b/cli/tests/test_journeys_report.py index 031da19..6bfe27c 100644 --- a/cli/tests/test_journeys_report.py +++ b/cli/tests/test_journeys_report.py @@ -118,3 +118,31 @@ def test_mcp_summary_lists_journeys_compactly(): assert s["journeys"][0]["breaks_at_step"] == "3. place order (POST /api/orders)" assert s["journeys"][1] == {"name": "old", "status": "stale", "reason": "step 1 (home): returned 404, expected 200"} + + +def test_a_confident_overall_gain_doesnt_make_a_noisy_journey_drop_confident(): + old = _run([_journey("browse", 100, 120, low=80, high=120)], survives=20) + new = _run([_journey("browse", 75, 90, low=70, high=110)], survives=40) + old["verdict"]["confidence"] = {"survives_low": 20, "survives_high": 20} + new["verdict"]["confidence"] = {"survives_low": 40, "survives_high": 40} + cmp = compare_results(old, new) + assert cmp["regressed"] and not cmp["confident"] and not cmp["improved"] + assert cmp["summary"] == ("Capacity regressed: browse 100 → 75 users (-25%). " + "(Overall: 20 → 40 users, +100%.)") + assert to_markdown(cmp).startswith("### 🟠") + + +def test_inconclusive_and_held_journeys_read_as_such(): + old = _run([_journey("browse", 200, None)]) + new = _run([_journey("browse", 50, None)]) + for run, tested in ((old, 200), (new, 50)): + run["journeys"][0]["max_tested"] = tested + md = to_markdown(compare_results(old, new)) + assert "| browse | 200+ | 50+ | inconclusive | — |" in md + + +def test_pipes_in_names_dont_break_the_markdown_table(): + old = _run([_journey("a|b", 40, 50, step="pay | ship")]) + new = _run([_journey("a|b", 20, 30, step="pay | ship")]) + md = to_markdown(compare_results(old, new)) + assert "| a\\|b | ~40 | ~20 | -50% 🔴 | pay \\| ship |" in md From 64e16aea3dee845d303e8206e55259c6a586aa7e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 21:41:49 +0000 Subject: [PATCH 09/11] feat(journeys): prescale ci and the Action replay journeys, with --stale --- CHANGELOG.md | 6 + README.md | 2 + action/README.md | 17 +++ action/action.yml | 15 ++ cli/src/prescale_cli/ci.py | 93 ++++++++++++- cli/src/prescale_cli/commands/ci.py | 154 ++++++++++++++++++--- cli/src/prescale_cli/commands/run.py | 27 ++-- cli/src/prescale_cli/journeys/load.py | 8 ++ cli/tests/test_action.py | 20 +++ cli/tests/test_ci_journeys.py | 191 ++++++++++++++++++++++++++ 10 files changed, 498 insertions(+), 35 deletions(-) create mode 100644 cli/tests/test_ci_journeys.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f9544..6b6cb92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,12 @@ `--expect`; the MCP tools' run summaries list each journey's verdict; and `prescale compare` diffs each journey by name (terminal, `--json`, `--markdown`). A drop past the threshold in any journey counts as a regression for `--fail-on-regression`, even when it isn't the weakest journey. +- **Journeys in CI.** `prescale ci` and the GitHub Action replay the checkout's journeys when + `prescale/journeys/` exists. The PR comment shows each journey's verdict, interference, and + journeys that no longer replay (listed always; `--stale fail` / `stale: fail` fails the check). + New Action inputs: `stale` and `allow-writes`. Until the dashboard compares journey runs only + with journey baselines, a journey run's PR comment shows no change instead of a possibly + misleading one, and never fails on it. - **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split across journeys by weight: each must hold its share alone, and all together must hold N. diff --git a/README.md b/README.md index 19d7d27..1515994 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,8 @@ Interference checkout broke with 15 of its own users, but held 20 alone. `prescale show`, `--html` reports and the MCP tools show journeys too, and `prescale compare` diffs each journey: a drop past the threshold in any journey is a regression (so `--fail-on-regression` catches it), even when it isn't the weakest one; it's marked confident when the confidence bands don't overlap. A journey that held everything it was tested to, below the old level, is inconclusive, not a drop. A URL run and a journey run measure different things, so comparing them is reported as not comparable. +In CI, `prescale ci` (and the GitHub Action) replays the checkout's journeys instead of URLs whenever `prescale/journeys/` exists, and the PR comment shows each journey's verdict, any interference, and journeys that no longer replay. Those are always listed; `--stale fail` (Action input `stale: fail`) also fails the check. With `app:` the replica is local, so journeys that change data can run; against a preview host they need `allow-writes`. Journey secrets (`{{env.NAME}}`) come from the step's `env:`. + `--no-combined` skips this pass. `--expect 500` says whether you'd hold 500 users at peak: in a journey run, each journey must hold its share of them (by weight) on its own, and all journeys together must hold 500; a journey that wasn't load-tested makes the answer unknown. It reports, it doesn't change the exit code (use `--fail-under` to gate CI). - `{{name}}` is a value an earlier step pulled out with `extract` (`body.` or `header.`); `{{unique.email}}` and `{{unique.id}}` are fresh each run; `{{env.NAME}}` reads an environment variable. diff --git a/action/README.md b/action/README.md index cf589e5..6de4a7b 100644 --- a/action/README.md +++ b/action/README.md @@ -78,6 +78,21 @@ jobs: fail-on-regression: true ``` +## Journeys + +If the checkout has journey files (`prescale/journeys/*.yaml`, see the main README), the action replays them at load instead of testing URLs: each journey alone, then all together. The PR comment shows each journey's verdict, any interference, and journeys that no longer replay against the build. Give journeys their secrets through the step's `env:` (for `{{env.NAME}}`): + +```yaml + - uses: pyjeebz/prescale/action@v1 + with: + token: ${{ secrets.PRESCALE_TOKEN }} + target: myapp.com + app: npm run build && npm start + stale: fail + env: + SHOP_PASSWORD: ${{ secrets.STAGING_TEST_PASSWORD }} +``` + ## How it works 1. A pull request is pushed (replica mode) or a deploy finishes (`deployment_status`). @@ -122,5 +137,7 @@ uploaded is the run's Result file, which you can inspect with | `app` | — | Replica mode: build-and-start command, tested at `url` (default `http://localhost:3000`) | | `app-timeout` | `600` | Seconds to wait for the app to answer | | `working-directory` | `.` | Where to run `app` (monorepos) | +| `stale` | `warn` | Journey runs: a journey that no longer replays is always listed on the PR; `fail` also fails the check | +| `allow-writes` | — | Journey runs: hosts (one per line) where journeys that change data may run; not needed with `app` | | `app-url` | `https://prescale.dev/dashboard` | Dashboard URL | | `prescale-version` | latest | CLI version | diff --git a/action/action.yml b/action/action.yml index 9fcf082..f7301bf 100644 --- a/action/action.yml +++ b/action/action.yml @@ -48,6 +48,14 @@ inputs: description: Confirm the deploy's host allows load testing. Vercel only permits it on Enterprise plans with notice, and Netlify and Cloudflare need it arranged; without this, PreScale skips their deploys and says why on the PR. required: false default: "false" + stale: + description: "Journey runs (when the checkout has prescale/journeys/): 'warn' or 'fail'. A journey that no longer replays against this build is always listed on the PR; 'fail' also fails the check." + required: false + default: "warn" + allow-writes: + description: "Journey runs: hosts (one per line) where journeys that change data may run, e.g. a staging host. Not needed with app: the replica is local. Journeys read their secrets from env variables you set on this step." + required: false + default: "" app-url: description: PreScale dashboard URL. required: false @@ -105,6 +113,8 @@ runs: INPUT_HOST_PERMISSION: ${{ inputs.host-permission }} INPUT_APP: ${{ inputs.app }} INPUT_APP_TIMEOUT: ${{ inputs.app-timeout }} + INPUT_STALE: ${{ inputs.stale }} + INPUT_ALLOW_WRITES: ${{ inputs.allow-writes }} run: | args=(--max-users "$INPUT_MAX_USERS" --stage-seconds "$INPUT_STAGE_SECONDS" --store "$RUNNER_TEMP/prescale") [ "$INPUT_FAIL" = "true" ] && args+=(--fail-on-regression) @@ -112,5 +122,10 @@ runs: [ "$INPUT_ALLOW_PRODUCTION" = "true" ] && args+=(--allow-production) [ "$INPUT_HOST_PERMISSION" = "true" ] && args+=(--host-permission) [ -n "$INPUT_APP" ] && args+=(--app "$INPUT_APP" --app-timeout "$INPUT_APP_TIMEOUT") + args+=(--stale "$INPUT_STALE") + while IFS= read -r host; do + host="${host//[[:space:]]/}" + [ -n "$host" ] && args+=(--allow-writes "$host") + done <<< "$INPUT_ALLOW_WRITES" [ -n "$INPUT_URL" ] && args+=("$INPUT_URL") prescale ci "${args[@]}" diff --git a/cli/src/prescale_cli/ci.py b/cli/src/prescale_cli/ci.py index dfc8385..00921db 100644 --- a/cli/src/prescale_cli/ci.py +++ b/cli/src/prescale_cli/ci.py @@ -181,11 +181,83 @@ def render_host_skip(rule, *, target: str) -> str: ]) +def comparison_for(result: dict, upload: dict) -> dict | None: + """The dashboard's comparison, if it compared like with like. A journey run is + only comparable with a journey baseline; a dashboard that doesn't say which + kind of run it compared against (`mode`) may have used a URL run, so its + comparison isn't trusted for journey runs (never a false regression).""" + comparison = upload.get("comparison") + if not comparison: + return None + if _mode(result) == "journeys" and comparison.get("mode") != "journeys": + return None + return comparison + + +def _mode(result: dict) -> str: + return (result.get("config") or {}).get("mode") or "urls" + + +def _cell(text) -> str: + return str(text).replace("|", "\\|").replace("\n", " ") + + +def render_journeys(result: dict) -> list[str]: + """Journey runs: each journey's verdict, the stale ones, and interference.""" + journeys = result.get("journeys") or [] + if not journeys: + return [] + lines = ["", "**Journeys** (each ramped on its own)", "", + "| journey | survives | breaks at step | why |", "|---|---:|---|---|"] + stale = [] + for j in journeys: + if j["status"] != "ok": + stale.append(j) + continue + survives = (f"{j['max_tested']}+ (held)" if j["onset_users"] is None + else f"~{j['survives_users']}") + step = j.get("culprit_step") + where = f"{step['index']}. {_cell(step['name'])} `{_cell(step['route'])}`" if step else "—" + lines.append(f"| {_cell(j['name'])} | {survives} | {where} | " + f"{_cell(j.get('bottleneck') or '')} |") + if stale: + lines += ["", "**Not load-tested** (they don't replay against this build, so the app " + "changed or it's broken; update the journey files if the change was on " + "purpose):"] + lines += [f"- ⚠️ **{_cell(j['name'])}** ({j['status']}): {_cell(j.get('reason') or '')}" + for j in stale] + inter = (result.get("combined") or {}).get("interference") + if inter: + lines += ["", f"**Interference:** {_cell(inter['journey'])} broke with " + f"{inter['users_at_break']} of its own users, but held " + f"{inter['held_alone']} alone. It competes with " + f"{_cell(', '.join(inter['with']))} for something they share " + "(a database pool, CPU, a cache)."] + return lines + + +def render_stale_only(runs, *, target: str) -> str: + """The PR comment when no journey replays, so nothing was load-tested.""" + lines = [comment_marker(target), "### ⚠️ PreScale: no journey replays against this build", "", + "Every journey failed with a single user, so there's no capacity to measure. " + "If the app changed on purpose, update the journey files (`prescale journeys " + "verify` shows what changed); if not, it's a bug.", ""] + for run in runs: + failed = run.check.failed + index = run.check.steps.index(failed) + 1 if failed else "?" + what = "stale" if run.check.stale else "couldn't run" + lines.append(f"- **{_cell(run.journey.name)}** ({what}): step {index} " + f"({_cell(failed.step.name if failed else '?')}): " + f"{_cell(failed.problem if failed else '')}") + return "\n".join(lines) + "\n" + + def render_comment(result: dict, upload: dict, *, target: str, sha: str | None) -> str: """The PR comment: the change vs baseline, what breaks first, why, the fix.""" verdict = result["verdict"] conf = verdict.get("confidence") or {} - comparison = upload.get("comparison") + comparison = comparison_for(result, upload) + unmatched = comparison is None and bool(upload.get("comparison")) target_info = upload.get("target") or {} baseline_branch = target_info.get("baselineBranch") or "main" tested_host = result["target"].get("host") or result["target"]["url"] @@ -238,13 +310,20 @@ def render_comment(result: dict, upload: dict, *, target: str, sha: str | None) lines += ["", "The confidence bands overlap, so this may be noise. Another run " "will confirm or clear it."] else: - lines.append(f"### ⚪ First capacity run: survives {survives}") + title = "Capacity" if unmatched else "First capacity run" + lines.append(f"### ⚪ {title}: survives {survives}") lines.append(f"`{tested_host}` · **{target}**") if rps: lines += ["", f"That's ≈ {rps:,.0f} req/s, roughly {visitors_for(rps):,} real " f"visitors. Visitors assume {VISITOR_NOTE}."] - lines += ["", f"No `{baseline_branch}` baseline yet. Later pushes to this PR compare " - f"with this run; once `{baseline_branch}` has a run, PRs compare with it."] + if unmatched: + lines += ["", "This run replayed journeys. The baseline it would be compared with " + "may have tested URLs, which measures something else, so no change is " + "shown."] + else: + lines += ["", f"No `{baseline_branch}` baseline yet. Later pushes to this PR " + f"compare with this run; once `{baseline_branch}` has a run, PRs " + "compare with it."] investigation = result.get("investigation") lines.append("") @@ -252,7 +331,9 @@ def render_comment(result: dict, upload: dict, *, target: str, sha: str | None) lines.append(f"**Held:** {_onset(verdict)} without crossing a failure threshold.") else: culprit = verdict.get("culprit_route") - lines.append(f"**Breaks first:** `{culprit}` · {_onset(verdict)}" if culprit + journey = verdict.get("culprit_journey") + where = f"{_cell(journey)} › `{culprit}`" if journey and culprit else f"`{culprit}`" + lines.append(f"**Breaks first:** {where} · {_onset(verdict)}" if culprit else f"**Breaks first:** {_onset(verdict)}") if investigation: lines.append(f"**Why:** `{investigation['bottleneck_class']}` " @@ -267,6 +348,8 @@ def render_comment(result: dict, upload: dict, *, target: str, sha: str | None) caption = f"p95 per route at {stage['users']} users · ┆ = the {wall:g}s latency wall" lines += ["", f"{caption}", "", "```text", *route_rows(stage, wall), "```"] + lines += render_journeys(result) + links = [f"[Open the run]({upload['url']})"] if upload.get("url") else [] if target_info.get("url"): links.append(f"[Capacity timeline]({target_info['url']})") diff --git a/cli/src/prescale_cli/commands/ci.py b/cli/src/prescale_cli/commands/ci.py index 197a9e0..e348c84 100644 --- a/cli/src/prescale_cli/commands/ci.py +++ b/cli/src/prescale_cli/commands/ci.py @@ -1,7 +1,8 @@ """`prescale ci` - capacity-check a deploy from CI and report it on the PR. Built for GitHub Actions (the PreScale action wraps it), usable from any CI: -investigate the deploy URL, upload the Result, compare it with the baseline +investigate the deploy URL (or, when the checkout has prescale/journeys/, +replay the journeys at load), upload the Result, compare it with the baseline (server-side, the same rule as the dashboard), write a job summary, and keep one capacity comment on the PR up to date. """ @@ -18,18 +19,27 @@ from prescale_cli.ci import ( comment_marker, + comparison_for, find_pull_request, github_context, render_comment, render_host_skip, + render_stale_only, upsert_pr_comment, ) from prescale_cli.cloud import CloudError, parse_headers, require_credentials, upload_result +from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica +from prescale_cli.commands.run import run_journeys from prescale_cli.hosts import PERMISSION_ENV from prescale_cli.investigate import investigate as run_investigate +from prescale_cli.journeys.load import NoJourneyReplaysError +from prescale_cli.journeys.model import load_dir from prescale_cli.live import ramp_progress -from prescale_cli.loadtest import HostPolicyError, LoadError, default_levels +from prescale_cli.loadtest import HostPolicyError, LoadError, default_levels, probe_host +from prescale_cli.result import write_result + +_LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"} console = Console() @@ -67,13 +77,22 @@ def _annotate(level: str, message: str) -> None: @click.option("--host-permission", "host_permission", is_flag=True, envvar=PERMISSION_ENV, help="Confirm the deploy's host allows load testing (Vercel, Netlify and " f"Cloudflare restrict it; otherwise PreScale skips). Env: {PERMISSION_ENV}=1.") +@click.option("--stale", "stale", type=click.Choice(["warn", "fail"]), default="warn", + show_default=True, + help="Journey runs: what a journey that no longer replays does to the check. It's " + "always reported on the PR; 'fail' also fails the job.") +@click.option("--allow-writes", "allow_writes", multiple=True, metavar="HOST", + help="Let journeys that change data run against this non-local host " + "(repeatable). Not needed with --app: the replica is local.") +@dir_option @app_option @app_timeout_option @click.option("--store", "store", type=click.Path(file_okay=False), default=None, help="Directory for saved runs (default: ./.prescale).") def ci(url: str | None, target: str, paths: tuple[str, ...], header_values: tuple[str, ...], max_users: int, stage_seconds: float, max_rps: float | None, fail_on_regression: bool, - comment: bool, allow_production: bool, host_permission: bool, app: str | None, + comment: bool, allow_production: bool, host_permission: bool, stale: str, + allow_writes: tuple[str, ...], directory: Path, app: str | None, app_timeout: float, store: str | None) -> None: """Capacity-check a deploy from CI, upload it, and comment on the PR. @@ -92,6 +111,9 @@ def ci(url: str | None, target: str, paths: tuple[str, ...], header_values: tupl \b Example (replica mode, on: pull_request): prescale ci --target myapp.com --app "npm run build && npm start" + + When prescale/journeys/ (or --dir) has journeys, they're replayed at load + instead of the URL routes, and the PR comment shows each journey's verdict. """ ctx = github_context() @@ -110,6 +132,20 @@ def ci(url: str | None, target: str, paths: tuple[str, ...], header_values: tupl if not url: raise click.UsageError("No URL to test. Pass one, or run on a deployment_status event.") + # A checkout with journey files replays them; without one (a deploy event with no + # checkout) it's a URL run as before. Broken files and missing secrets stop the + # job here, before an app is built. + journeys, errors = load_dir(directory) + for error in errors: + console.print(f"[red]Error:[/red] {error}", markup=False) + _annotate("error", f"PreScale: {error}") + if errors: + raise SystemExit(2) + if journeys: + host = (httpx.URL(url).host or "").lower() + refuse_unallowed_writes(journeys, host, {h.lower() for h in allow_writes}) + refuse_missing_env(journeys) + try: headers = parse_headers(header_values + tuple(_lines(os.environ.get("PRESCALE_HEADERS")))) creds = require_credentials() @@ -136,6 +172,14 @@ def ci(url: str | None, target: str, paths: tuple[str, ...], header_values: tupl if app: start_replica(app, url, timeout=app_timeout, console=console, quiet=False) + if journeys: + _ci_journeys(journeys, url, target=target, ctx=ctx, creds=creds, headers=headers, + max_users=max_users, stage_seconds=stage_seconds, max_rps=max_rps, + store=store, comment=comment, pr_number=pr_number, + github_token=github_token, host_permission=host_permission, + replica=bool(app), stale=stale, fail_on_regression=fail_on_regression) + return + extra_paths = tuple(paths) + tuple(_lines(os.environ.get("PRESCALE_PATHS"))) console.print(f"[bold]PreScale[/bold] — capacity check of [cyan]{url}[/cyan] for " f"[bold]{target}[/bold]" @@ -163,28 +207,71 @@ def ci(url: str | None, target: str, paths: tuple[str, ...], header_values: tupl _annotate("error", f"PreScale couldn't load-test {url}: {exc}") raise SystemExit(1) from None + if _report(result, url=url, target=target, ctx=ctx, creds=creds, comment=comment, + pr_number=pr_number, github_token=github_token, + fail_on_regression=fail_on_regression): + raise SystemExit(1) + + +def _ci_journeys(journeys, url: str, *, target: str, ctx, creds, headers: dict[str, str], + max_users: int, stage_seconds: float, max_rps: float | None, + store: str | None, comment: bool, pr_number: int | None, + github_token: str | None, host_permission: bool, replica: bool, stale: str, + fail_on_regression: bool) -> None: + """Replay the checkout's journeys at load, upload, comment, and gate.""" + # Journey ramps skip the preflight that checks the host, so check it here. + if not replica and not host_permission: + rule = asyncio.run(probe_host(url, headers=headers)) + if rule is not None: + _skip_restricted_host(rule, target=target, ctx=ctx, comment=comment, + pr_number=pr_number, github_token=github_token) + return + console.print(f"[bold]PreScale[/bold] — journey check for [bold]{target}[/bold]" + + (f" (PR #{pr_number})" if pr_number else "")) + config = {"method": "GET", "max_users": max_users, "stage_seconds": stage_seconds, + "latency_wall_s": 2.0, "error_threshold": 0.02, "max_rps": max_rps, + "warmup": True, "repeat": 1, "think_time_s": 0.0, "fail_under": None, + "profile": None} try: - upload = upload_result(result, creds, pr=pr_number, target=target) - except CloudError as exc: - console.print(f"[red]✗ upload failed:[/red] {exc}") - _annotate("error", f"PreScale upload failed: {exc}") + _, result, _ = run_journeys(journeys, url, config, latency_wall=2.0, + error_threshold=0.02, timeout=10.0, headers=headers, + replica=replica, as_json=False) + except NoJourneyReplaysError as exc: + body = render_stale_only(exc.runs, target=target) + _publish(body, target=target, ctx=ctx, comment=comment, pr_number=pr_number, + github_token=github_token) + level = "error" if stale == "fail" else "warning" + _annotate(level, "PreScale: no journey replays against this build, so nothing was " + "load-tested. Update the journey files if the app changed on purpose.") + if stale == "fail": + raise SystemExit(1) + return + except LoadError as exc: + console.print(f"[red]Error:[/red] {exc}") + _annotate("error", f"PreScale couldn't load-test {url}: {exc}") raise SystemExit(1) from None + write_result(result, store=store) + + stale_names = [j["name"] for j in result["journeys"] if j["status"] != "ok"] + regressed = _report(result, url=url, target=target, ctx=ctx, creds=creds, comment=comment, + pr_number=pr_number, github_token=github_token, + fail_on_regression=fail_on_regression) + if stale_names: + level = "error" if stale == "fail" else "warning" + what = ("replays against this build, so it wasn't" if len(stale_names) == 1 + else "replay against this build, so they weren't") + _annotate(level, f"PreScale: {', '.join(stale_names)} no longer {what} load-tested.") + if regressed or (stale_names and stale == "fail"): + raise SystemExit(1) - body = render_comment(result, upload, target=target, sha=ctx.sha) - comparison = upload.get("comparison") - verdict = result["verdict"] - sent = sum(stage.get("total", 0) for stage in result.get("stages", [])) - console.print(f"[dim]Sent {sent:,} requests while measuring.[/dim]") - console.print(f"survives ~{verdict['survives_users']} users" - + (f" · {comparison['kind']} {round(comparison['pct'] * 100):+d}% vs " - f"{comparison['against']}" if comparison else " · first run") - + f"\n[green]✓[/green] uploaded → [cyan]{upload.get('url')}[/cyan]") +def _publish(body: str, *, target: str, ctx, comment: bool, pr_number: int | None, + github_token: str | None) -> None: + """The job summary, and the one PreScale comment on the PR.""" summary_path = os.environ.get("GITHUB_STEP_SUMMARY") if summary_path: with Path(summary_path).open("a", encoding="utf-8") as fh: fh.write(body + "\n") - if comment and pr_number and github_token and ctx.repository: try: action = upsert_pr_comment(ctx, github_token, pr_number, body, comment_marker(target)) @@ -195,15 +282,42 @@ def ci(url: str | None, target: str, paths: tuple[str, ...], header_values: tupl elif comment and not pr_number: console.print("[dim]No open PR for this commit, so no PR comment.[/dim]") + +def _report(result: dict, *, url: str, target: str, ctx, creds, comment: bool, + pr_number: int | None, github_token: str | None, + fail_on_regression: bool) -> bool: + """Upload the run, comment on the PR, and annotate a regression. Returns + whether the job should fail (a regression with --fail-on-regression); the + caller exits, after any notes of its own.""" + try: + upload = upload_result(result, creds, pr=pr_number, target=target) + except CloudError as exc: + console.print(f"[red]✗ upload failed:[/red] {exc}") + _annotate("error", f"PreScale upload failed: {exc}") + raise SystemExit(1) from None + + body = render_comment(result, upload, target=target, sha=ctx.sha) + comparison = comparison_for(result, upload) + verdict = result["verdict"] + sent = sum(stage.get("total", 0) for stage in result.get("stages", [])) + console.print(f"[dim]Sent {sent:,} requests while measuring.[/dim]") + console.print(f"survives ~{verdict['survives_users']} users" + + (f" · {comparison['kind']} {round(comparison['pct'] * 100):+d}% vs " + f"{comparison['against']}" if comparison else " · first run") + + f"\n[green]✓[/green] uploaded → [cyan]{upload.get('url')}[/cyan]") + + _publish(body, target=target, ctx=ctx, comment=comment, pr_number=pr_number, + github_token=github_token) + kind = comparison["kind"] if comparison else None if kind == "regressed": _annotate("error", f"Capacity regressed {round(comparison['pct'] * 100)}% vs " f"{comparison['against']}.") - if fail_on_regression: - raise SystemExit(1) - elif kind == "possible": + return fail_on_regression + if kind == "possible": _annotate("warning", f"Possible capacity regression {round(comparison['pct'] * 100)}% " f"vs {comparison['against']} (bands overlap).") + return False def _skip_restricted_host(rule, *, target: str, ctx, comment: bool, pr_number: int | None, diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index 0dbe69c..047f8bf 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -23,6 +23,7 @@ from prescale_cli.commands.upload import header_option, upload_after_run, upload_option from prescale_cli.journeys.load import ( JourneyRun, + NoJourneyReplaysError, combined_block, first_to_break, journeys_block, @@ -205,10 +206,16 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, as_json=as_json, upload=upload, fail_under=fail_under, expect=expect) if journeys: - report, result, live_mode = _run_journeys( - journeys, url, config, latency_wall=latency_wall, - error_threshold=error_threshold, timeout=timeout, headers=headers, - replica=bool(app), as_json=as_json, combined=not no_combined) + try: + report, result, live_mode = run_journeys( + journeys, url, config, latency_wall=latency_wall, + error_threshold=error_threshold, timeout=timeout, headers=headers, + replica=bool(app), as_json=as_json, combined=not no_combined) + except NoJourneyReplaysError: + (Console(stderr=True) if as_json else console).print( + "[red]Error:[/red] no journey replays against this app, so there's nothing " + "to load-test. Run `prescale journeys verify` for details.") + raise SystemExit(1) _finish(result, report, live_mode=live_mode, **finish) return @@ -373,12 +380,14 @@ def _journeys_to_run(directory: Path, only: tuple[str, ...], *, urls_only: bool, return found -def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wall: float, +def run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wall: float, error_threshold: float, timeout: float, headers: dict[str, str], replica: bool, as_json: bool, combined: bool = True) -> tuple: """The isolation pass: check each journey once, then ramp each one that - replays on its own. Returns (report, result, live_mode) for the first - journey to break, with every journey's verdict in result["journeys"].""" + replays on its own; then, with two or more, the combined pass. Returns + (report, result, live_mode) for the first journey to break, with every + journey's verdict in result["journeys"]. Raises NoJourneyReplaysError if every + journey is stale or unreachable. Shared by `run` and `ci`.""" if not as_json: console.print(f"\n[bold]PreScale[/bold] — replaying {len(journeys)} " f"journey{'s' if len(journeys) != 1 else ''} against [cyan]{url}[/cyan]") @@ -418,9 +427,7 @@ def _run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wa worst = first_to_break(runs) if worst is None: - out.print("[red]Error:[/red] no journey replays against this app, so there's " - "nothing to load-test. Run `prescale journeys verify` for details.") - raise SystemExit(1) + raise NoJourneyReplaysError(runs) warnings = list(dict.fromkeys(r.warning for r in runs if r.warning)) result = build_result(worst.report, url=url, targets=[s.label for s in worst.journey.steps], diff --git a/cli/src/prescale_cli/journeys/load.py b/cli/src/prescale_cli/journeys/load.py index 9cd24a8..bd03a9e 100644 --- a/cli/src/prescale_cli/journeys/load.py +++ b/cli/src/prescale_cli/journeys/load.py @@ -28,6 +28,14 @@ from prescale_cli.loadtest import RunReport, StageResult, run_loadtest +class NoJourneyReplaysError(Exception): + """Every journey was stale or unreachable, so nothing could be load-tested.""" + + def __init__(self, runs: list[JourneyRun]) -> None: + super().__init__("no journey replays against this app") + self.runs = runs + + @dataclass class JourneyRun: journey: Journey diff --git a/cli/tests/test_action.py b/cli/tests/test_action.py index 06b4c4f..c44682d 100644 --- a/cli/tests/test_action.py +++ b/cli/tests/test_action.py @@ -15,3 +15,23 @@ def test_metadata_has_no_expressions_outside_defaults_and_steps(): for name, spec in action["inputs"].items(): assert "${{" not in spec["description"], name assert "secrets." not in str(spec.get("default", "")), name + + +def test_capacity_check_passes_stale_and_each_allow_writes_host(tmp_path): + import os + import subprocess + + stub = tmp_path / "prescale" + stub.write_text('#!/bin/sh\nfor a in "$@"; do printf "[%s]" "$a"; done\n') + stub.chmod(0o755) + action = yaml.safe_load(ACTION.read_text()) + script = next(s for s in action["runs"]["steps"] if s.get("name") == "Capacity check")["run"] + env = {"PATH": f"{tmp_path}:{os.environ['PATH']}", "RUNNER_TEMP": "/tmp", + "INPUT_MAX_USERS": "100", "INPUT_STAGE_SECONDS": "5", "INPUT_FAIL": "false", + "INPUT_COMMENT": "true", "INPUT_ALLOW_PRODUCTION": "false", + "INPUT_HOST_PERMISSION": "false", "INPUT_APP": "", "INPUT_APP_TIMEOUT": "600", + "INPUT_URL": "", "INPUT_STALE": "fail", + "INPUT_ALLOW_WRITES": "staging.myapp.com\n qa.myapp.com \n\n"} + out = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True).stdout + assert out.endswith("[--stale][fail][--allow-writes][staging.myapp.com]" + "[--allow-writes][qa.myapp.com]") diff --git a/cli/tests/test_ci_journeys.py b/cli/tests/test_ci_journeys.py new file mode 100644 index 0000000..95d285f --- /dev/null +++ b/cli/tests/test_ci_journeys.py @@ -0,0 +1,191 @@ +"""`prescale ci` with journeys: the comment, stale handling, and the safe comparison.""" + +import json + +import pytest +from click.testing import CliRunner + +from prescale_cli import ci as ci_mod +from prescale_cli.journeys.load import JourneyRun, NoJourneyReplaysError +from prescale_cli.journeys.model import parse +from prescale_cli.journeys.replay import JourneyCheck, StepCheck +from prescale_cli.main import cli + +CHECKOUT = {"version": 1, "name": "checkout", "type": "checkout", + "steps": [{"name": "place order", "request": {"method": "POST", "path": "/api/orders"}}]} +BROWSE = {"version": 1, "name": "browse", "steps": [{"name": "home", "request": {"path": "/"}}]} + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch, tmp_path): + for name in ("GITHUB_ACTIONS", "GITHUB_EVENT_NAME", "GITHUB_EVENT_PATH", "GITHUB_REPOSITORY", + "GITHUB_SHA", "GITHUB_REF", "GITHUB_HEAD_REF", "GITHUB_TOKEN", + "GITHUB_STEP_SUMMARY", "PRESCALE_HEADERS", "PRESCALE_PATHS", + "PRESCALE_GIT_COMMIT", "PRESCALE_GIT_BRANCH", "PRESCALE_TARGET"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("PRESCALE_CONFIG_DIR", str(tmp_path / "config")) + monkeypatch.setenv("PRESCALE_TOKEN", "psk_test") + monkeypatch.setenv("GITHUB_ACTIONS", "true") + + +def _journey_result(stale=False): + journeys = [ + {"name": "checkout", "type": "checkout", "weight": 5, "status": "ok", "reason": None, + "survives_users": 20, "max_tested": 100, "onset_users": 30, "onset_reason": "errors", + "confidence": {"survives_low": 20, "survives_high": 30, "stable": False}, + "culprit_step": {"index": 1, "name": "place order", "route": "POST /api/orders"}, + "bottleneck": "5xx under load"}, + {"name": "browse", "type": "browse", "weight": 60, + "status": "stale" if stale else "ok", + "reason": "step 1 (home): returned 404, expected a status below 400" if stale else None, + **({} if stale else {"survives_users": 100, "max_tested": 100, "onset_users": None, + "onset_reason": None, "confidence": {}, "culprit_step": None, + "bottleneck": None})}, + ] + return { + "id": "r1", "target": {"url": "http://localhost:3000", "host": "localhost"}, + "config": {"mode": "journeys", "replica": True}, + "verdict": {"survives_users": 20, "max_tested": 100, "onset_users": 30, + "onset_reason": "errors", "culprit_route": "POST /api/orders", + "culprit_journey": "checkout", "peak_rps": 120.0, + "confidence": {"survives_low": 20, "survives_high": 30, "stable": False}}, + "environment": {"git_commit": "a41f9c2"}, + "journeys": journeys, + "combined": {"interference": {"journey": "checkout", "users_at_break": 15, + "held_alone": 20, "with": ["browse"]}}, + } + + +def _upload(mode=None, kind="regressed"): + comparison = {"kind": kind, "pct": -0.5, "confident": True, "against": "main @ 71aa0c9", + "baseline": {"survives": 40, "low": 35, "high": 45, "peakRps": 200.0, + "commit": "71aa0c9"}} + if mode: + comparison["mode"] = mode + return {"url": "https://prescale.dev/dashboard/runs/x", "comparison": comparison, + "target": {"baselineBranch": "main", "verified": True}} + + +# --- the comment ----------------------------------------------------------------- + +def test_a_journey_run_ignores_a_comparison_that_may_be_against_a_url_run(): + assert ci_mod.comparison_for(_journey_result(), _upload()) is None + assert ci_mod.comparison_for(_journey_result(), _upload(mode="journeys"))["kind"] == "regressed" + url_run = {**_journey_result(), "config": {}} + assert ci_mod.comparison_for(url_run, _upload())["kind"] == "regressed" + + body = ci_mod.render_comment(_journey_result(), _upload(), target="myapp.com", sha=None) + assert "Capacity regressed" not in body + assert "### ⚪ Capacity: survives ~20 users" in body + assert "may have tested URLs" in body + + +def test_journey_comment_lists_journeys_stale_ones_and_interference(): + body = ci_mod.render_comment(_journey_result(stale=True), _upload(mode="journeys"), + target="myapp.com", sha=None) + assert "### 🔴" in body or "Capacity regressed" in body + assert "**Breaks first:** checkout › `POST /api/orders`" in body + assert "| checkout | ~20 | 1. place order `POST /api/orders` | 5xx under load |" in body + assert "- ⚠️ **browse** (stale): step 1 (home): returned 404" in body + assert "checkout broke with 15 of its own users, but held 20 alone" in body + + +def test_stale_only_comment(): + runs = [] + for doc in (CHECKOUT, BROWSE): + j = parse(doc) + c = JourneyCheck(j) + c.steps = [StepCheck(j.steps[0], status=404, problem="returned 404, expected 201")] + runs.append(JourneyRun(j, c)) + body = ci_mod.render_stale_only(runs, target="myapp.com") + assert body.startswith(ci_mod.comment_marker("myapp.com")) + assert "no journey replays against this build" in body + assert "- **checkout** (stale): step 1 (place order): returned 404, expected 201" in body + + +# --- the command ------------------------------------------------------------------ + +def _dir(tmp_path): + d = tmp_path / "journeys" + d.mkdir(exist_ok=True) + for doc in (CHECKOUT, BROWSE): + (d / f"{doc['name']}.yaml").write_text(json.dumps(doc)) + return str(d) + + +def _wire(monkeypatch, *, result=None, raises=None, upload=None): + seen = {} + + def fake_run_journeys(journeys, url, config, **kwargs): + seen["journeys"] = [j.name for j in journeys] + seen["url"], seen["replica"] = url, kwargs["replica"] + if raises: + raise raises + return None, result, False + + def fake_upsert(ctx, token, pr_number, body, marker): + seen["comment"] = body + return "created" + + monkeypatch.setattr("prescale_cli.commands.ci.run_journeys", fake_run_journeys) + monkeypatch.setattr("prescale_cli.commands.ci.upload_result", + lambda *a, **k: upload or _upload()) + monkeypatch.setattr("prescale_cli.commands.ci.upsert_pr_comment", fake_upsert) + monkeypatch.setattr("prescale_cli.commands.ci.find_pull_request", + lambda ctx, token: {"number": 7, "head": {"ref": "feat/x"}}) + monkeypatch.setattr("prescale_cli.commands.ci.run_investigate", + lambda *a, **k: pytest.fail("journey runs don't investigate URLs")) + + async def no_rule(url, **kwargs): + return None + monkeypatch.setattr("prescale_cli.commands.ci.probe_host", no_rule) + monkeypatch.setenv("GITHUB_TOKEN", "ghs_x") + monkeypatch.setenv("GITHUB_REPOSITORY", "acme/shop") + return seen + + +def _ci(tmp_path, *extra): + return CliRunner().invoke(cli, ["ci", "http://localhost:3000", "--target", "myapp.com", + "--dir", _dir(tmp_path), "--store", str(tmp_path / "s"), + *extra]) + + +def test_ci_replays_journeys_and_warns_on_stale(monkeypatch, tmp_path): + seen = _wire(monkeypatch, result=_journey_result(stale=True)) + res = _ci(tmp_path) + assert res.exit_code == 0, res.output + assert seen["journeys"] == ["browse", "checkout"] + assert "::warning::PreScale: browse no longer replays" in res.output + assert "Not load-tested" in seen["comment"] + + +def test_ci_stale_fail_fails_the_job(monkeypatch, tmp_path): + _wire(monkeypatch, result=_journey_result(stale=True)) + res = _ci(tmp_path, "--stale", "fail") + assert res.exit_code == 1 + assert "::error::PreScale: browse no longer replays" in res.output + + +def test_ci_never_fails_a_journey_run_on_an_unmarked_comparison(monkeypatch, tmp_path): + _wire(monkeypatch, result=_journey_result()) + assert _ci(tmp_path, "--fail-on-regression").exit_code == 0 + _wire(monkeypatch, result=_journey_result(), upload=_upload(mode="journeys")) + assert _ci(tmp_path, "--fail-on-regression").exit_code == 1 + + +def test_ci_comments_when_no_journey_replays(monkeypatch, tmp_path): + seen = _wire(monkeypatch, raises=NoJourneyReplaysError([])) + res = _ci(tmp_path) + assert res.exit_code == 0 + assert "no journey replays against this build" in seen["comment"] + assert "::warning::PreScale: no journey replays" in res.output + _wire(monkeypatch, raises=NoJourneyReplaysError([])) + assert _ci(tmp_path, "--stale", "fail").exit_code == 1 + + +def test_ci_refuses_journey_writes_on_a_preview_host(monkeypatch, tmp_path): + _wire(monkeypatch, result=_journey_result()) + res = CliRunner().invoke(cli, ["ci", "https://preview.example.com", "--target", "myapp.com", + "--dir", _dir(tmp_path)]) + assert res.exit_code == 2 + assert "--allow-writes preview.example.com" in " ".join(res.output.split()) From 1a2e859f5599a61f6e93bf4699eb962058aa088c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 21:51:04 +0000 Subject: [PATCH 10/11] fix(journeys): no response bodies in CI, down apps fail, Action stays compatible --- CHANGELOG.md | 6 +- action/README.md | 4 ++ action/action.yml | 8 ++- cli/src/prescale_cli/ci.py | 17 ++++-- cli/src/prescale_cli/commands/ci.py | 29 ++++++++-- cli/src/prescale_cli/commands/journeys.py | 18 +++--- cli/src/prescale_cli/commands/run.py | 44 +++++++------- cli/src/prescale_cli/journeys/load.py | 5 +- cli/src/prescale_cli/journeys/model.py | 4 +- cli/src/prescale_cli/journeys/replay.py | 57 +++++++++++++++---- cli/src/prescale_cli/render.py | 6 +- cli/src/prescale_cli/report.py | 6 +- .../prescale_cli/schema/result.schema.json | 2 +- cli/tests/test_action.py | 14 ++++- cli/tests/test_ci_journeys.py | 56 ++++++++++++++++++ cli/tests/test_journeys.py | 19 ++++++- cli/tests/test_journeys_load.py | 2 +- 17 files changed, 229 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b6cb92..df9e7f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,11 @@ - **Journeys in CI.** `prescale ci` and the GitHub Action replay the checkout's journeys when `prescale/journeys/` exists. The PR comment shows each journey's verdict, interference, and journeys that no longer replay (listed always; `--stale fail` / `stale: fail` fails the check). - New Action inputs: `stale` and `allow-writes`. Until the dashboard compares journey runs only + A journey that gets no answer, or a 5xx with a single user, always fails the check (that's the + app, not a stale file). Response bodies never reach the PR comment or uploads; locally, + `journeys verify` shows them with every secret the journeys read masked. + New Action inputs: `stale` and `allow-writes` (the action only passes them when set, so a + pinned older `prescale-version` keeps working). Until the dashboard compares journey runs only with journey baselines, a journey run's PR comment shows no change instead of a possibly misleading one, and never fails on it. - **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split diff --git a/action/README.md b/action/README.md index 6de4a7b..9ba1f23 100644 --- a/action/README.md +++ b/action/README.md @@ -93,6 +93,10 @@ If the checkout has journey files (`prescale/journeys/*.yaml`, see the main READ SHOP_PASSWORD: ${{ secrets.STAGING_TEST_PASSWORD }} ``` +A journey that **no longer replays** (the app answers differently, e.g. a 404) is listed on the PR; `stale: fail` also fails the check. A journey that gets **no answer, or a 5xx with a single user**, always fails the check: that's the app broken or down, not a stale file. Response bodies never appear in the PR comment. + +Deploy workflows (`on: deployment_status`) that also check out the code replay journeys too. Journeys that change data then need the preview's host in `allow-writes`; preview hosts change per deploy, so prefer replica mode (`app:`) for journeys that write. + ## How it works 1. A pull request is pushed (replica mode) or a deploy finishes (`deployment_status`). diff --git a/action/action.yml b/action/action.yml index f7301bf..ff1b6e0 100644 --- a/action/action.yml +++ b/action/action.yml @@ -53,7 +53,7 @@ inputs: required: false default: "warn" allow-writes: - description: "Journey runs: hosts (one per line) where journeys that change data may run, e.g. a staging host. Not needed with app: the replica is local. Journeys read their secrets from env variables you set on this step." + description: "Journey runs: hosts (one per line, or comma-separated) where journeys that change data may run, e.g. a staging host. Not needed with app: the replica is local. Journeys read their secrets from env variables you set on this step." required: false default: "" app-url: @@ -122,10 +122,12 @@ runs: [ "$INPUT_ALLOW_PRODUCTION" = "true" ] && args+=(--allow-production) [ "$INPUT_HOST_PERMISSION" = "true" ] && args+=(--host-permission) [ -n "$INPUT_APP" ] && args+=(--app "$INPUT_APP" --app-timeout "$INPUT_APP_TIMEOUT") - args+=(--stale "$INPUT_STALE") + # New flags only when set, so a pinned older prescale-version keeps working. + stale="${INPUT_STALE,,}" + [ -n "$stale" ] && [ "$stale" != "warn" ] && args+=(--stale "$stale") while IFS= read -r host; do host="${host//[[:space:]]/}" [ -n "$host" ] && args+=(--allow-writes "$host") - done <<< "$INPUT_ALLOW_WRITES" + done <<< "${INPUT_ALLOW_WRITES//,/$'\n'}" [ -n "$INPUT_URL" ] && args+=("$INPUT_URL") prescale ci "${args[@]}" diff --git a/cli/src/prescale_cli/ci.py b/cli/src/prescale_cli/ci.py index 00921db..08929e3 100644 --- a/cli/src/prescale_cli/ci.py +++ b/cli/src/prescale_cli/ci.py @@ -16,6 +16,7 @@ import httpx from prescale_cli.bars import bar_rows, route_rows +from prescale_cli.journeys.load import LABELS from prescale_cli.throughput import VISITOR_NOTE, capacity_rps, visitors_for _SHA = re.compile(r"^[0-9a-f]{40}$") @@ -199,7 +200,15 @@ def _mode(result: dict) -> str: def _cell(text) -> str: - return str(text).replace("|", "\\|").replace("\n", " ") + """Text from journey files or the app, safe in a PR comment: it can't break the + table, render HTML, or @mention (and so notify) anyone.""" + return (str(text).replace("|", "\\|").replace("\n", " ").replace("<", "<") + .replace(">", ">").replace("@", "@\u200b")) + + +def _code(text) -> str: + """TEXT as inline code that a backtick inside can't break out of.""" + return "`" + _cell(text).replace("`", "'") + "`" def render_journeys(result: dict) -> list[str]: @@ -217,7 +226,7 @@ def render_journeys(result: dict) -> list[str]: survives = (f"{j['max_tested']}+ (held)" if j["onset_users"] is None else f"~{j['survives_users']}") step = j.get("culprit_step") - where = f"{step['index']}. {_cell(step['name'])} `{_cell(step['route'])}`" if step else "—" + where = f"{step['index']}. {_cell(step['name'])} {_code(step['route'])}" if step else "—" lines.append(f"| {_cell(j['name'])} | {survives} | {where} | " f"{_cell(j.get('bottleneck') or '')} |") if stale: @@ -245,7 +254,7 @@ def render_stale_only(runs, *, target: str) -> str: for run in runs: failed = run.check.failed index = run.check.steps.index(failed) + 1 if failed else "?" - what = "stale" if run.check.stale else "couldn't run" + what = LABELS[run.check.outcome] lines.append(f"- **{_cell(run.journey.name)}** ({what}): step {index} " f"({_cell(failed.step.name if failed else '?')}): " f"{_cell(failed.problem if failed else '')}") @@ -332,7 +341,7 @@ def render_comment(result: dict, upload: dict, *, target: str, sha: str | None) else: culprit = verdict.get("culprit_route") journey = verdict.get("culprit_journey") - where = f"{_cell(journey)} › `{culprit}`" if journey and culprit else f"`{culprit}`" + where = f"{_cell(journey)} › {_code(culprit)}" if journey and culprit else _code(culprit) lines.append(f"**Breaks first:** {where} · {_onset(verdict)}" if culprit else f"**Breaks first:** {_onset(verdict)}") if investigation: diff --git a/cli/src/prescale_cli/commands/ci.py b/cli/src/prescale_cli/commands/ci.py index e348c84..1626655 100644 --- a/cli/src/prescale_cli/commands/ci.py +++ b/cli/src/prescale_cli/commands/ci.py @@ -39,8 +39,6 @@ from prescale_cli.loadtest import HostPolicyError, LoadError, default_levels, probe_host from prescale_cli.result import write_result -_LOCAL_HOSTS = {"localhost", "127.0.0.1", "0.0.0.0", "::1"} - console = Console() @@ -143,8 +141,16 @@ def ci(url: str | None, target: str, paths: tuple[str, ...], header_values: tupl raise SystemExit(2) if journeys: host = (httpx.URL(url).host or "").lower() - refuse_unallowed_writes(journeys, host, {h.lower() for h in allow_writes}) - refuse_missing_env(journeys) + try: + refuse_unallowed_writes(journeys, host, {h.lower() for h in allow_writes}) + refuse_missing_env(journeys) + except SystemExit: + _annotate("error", "PreScale refused to replay the journeys (see the log: " + "journeys that change data need allow-writes for this host, " + "and every {{env.NAME}} they read must be set).") + raise + if paths or os.environ.get("PRESCALE_PATHS"): + console.print("[dim]Journeys are replayed, so --path / paths are ignored.[/dim]") try: headers = parse_headers(header_values + tuple(_lines(os.environ.get("PRESCALE_HEADERS")))) @@ -240,6 +246,13 @@ def _ci_journeys(journeys, url: str, *, target: str, ctx, creds, headers: dict[s body = render_stale_only(exc.runs, target=target) _publish(body, target=target, ctx=ctx, comment=comment, pr_number=pr_number, github_token=github_token) + # Only "stale" (the app answered differently) follows --stale. An app that + # doesn't answer, or fails with a single user, fails the check like URL mode. + broken = [r.journey.name for r in exc.runs if r.check.outcome != "stale"] + if broken: + _annotate("error", f"PreScale: {', '.join(broken)} failed with a single user " + "(no answer, or a 5xx), so nothing was load-tested.") + raise SystemExit(1) level = "error" if stale == "fail" else "warning" _annotate(level, "PreScale: no journey replays against this build, so nothing was " "load-tested. Update the journey files if the app changed on purpose.") @@ -252,7 +265,8 @@ def _ci_journeys(journeys, url: str, *, target: str, ctx, creds, headers: dict[s raise SystemExit(1) from None write_result(result, store=store) - stale_names = [j["name"] for j in result["journeys"] if j["status"] != "ok"] + stale_names = [j["name"] for j in result["journeys"] if j["status"] == "stale"] + broken = [j["name"] for j in result["journeys"] if j["status"] in ("error", "unreachable")] regressed = _report(result, url=url, target=target, ctx=ctx, creds=creds, comment=comment, pr_number=pr_number, github_token=github_token, fail_on_regression=fail_on_regression) @@ -261,7 +275,10 @@ def _ci_journeys(journeys, url: str, *, target: str, ctx, creds, headers: dict[s what = ("replays against this build, so it wasn't" if len(stale_names) == 1 else "replay against this build, so they weren't") _annotate(level, f"PreScale: {', '.join(stale_names)} no longer {what} load-tested.") - if regressed or (stale_names and stale == "fail"): + if broken: + _annotate("error", f"PreScale: {', '.join(broken)} failed with a single user (no " + "answer, or a 5xx), so they weren't load-tested.") + if regressed or broken or (stale_names and stale == "fail"): raise SystemExit(1) diff --git a/cli/src/prescale_cli/commands/journeys.py b/cli/src/prescale_cli/commands/journeys.py index 5918f59..f5cb898 100644 --- a/cli/src/prescale_cli/commands/journeys.py +++ b/cli/src/prescale_cli/commands/journeys.py @@ -96,12 +96,16 @@ def verify(url: str, directory: Path, only: tuple[str, ...], allow_writes: tuple broken = [r for r in results if not r.ok] if broken: - stale = [r.journey.name for r in broken if r.stale] + stale = [r.journey.name for r in broken if r.outcome == "stale"] if stale: console.print(f"\n[yellow]{len(stale)} stale:[/yellow] {', '.join(stale)}. The app " "no longer answers the way these journeys expect. If the change was " "on purpose, update the journey files; if not, it's a bug.") - unreached = [r.journey.name for r in broken if not r.stale] + errors = [r.journey.name for r in broken if r.outcome == "error"] + if errors: + console.print(f"\n[red]{len(errors)} failing with a 5xx:[/red] {', '.join(errors)}. " + "With no load, that's a bug in the app, not a stale journey.") + unreached = [r.journey.name for r in broken if r.outcome == "unreachable"] if unreached: console.print(f"\n[red]{len(unreached)} couldn't run:[/red] {', '.join(unreached)}.") raise SystemExit(1) @@ -151,8 +155,8 @@ async def _check_all(found: list[Journey], base: str, *, timeout: float, def _print_check(result: JourneyCheck) -> None: journey = result.journey - mark = "[green]✓[/green]" if result.ok else ("[yellow]stale[/yellow]" if result.stale - else "[red]✗[/red]") + mark = {"ok": "[green]✓[/green]", "stale": "[yellow]stale[/yellow]"}.get( + result.outcome, "[red]✗[/red]") console.print(f"\n{mark} [bold]{escape(journey.name)}[/bold] [dim]({journey.type}, " f"{len(journey.steps)} steps)[/dim]") for i, step in enumerate(journey.steps, 1): @@ -166,6 +170,6 @@ def _print_check(result: JourneyCheck) -> None: else: console.print(f" {i}. {name} [dim]{label}[/dim] " f"[red]{escape(done.problem or '')}[/red]", highlight=False) - if done.status is not None and done.status >= 500: - console.print(" [dim]A 5xx with no load is usually a bug in the app, " - "not a stale journey.[/dim]") + if done.snippet: + # Local terminal only, secrets masked; never in CI output or uploads. + console.print(f" [dim]{escape(done.snippet)}[/dim]", highlight=False) diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index 047f8bf..d3da268 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -22,6 +22,7 @@ from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica from prescale_cli.commands.upload import header_option, upload_after_run, upload_option from prescale_cli.journeys.load import ( + LABELS, JourneyRun, NoJourneyReplaysError, combined_block, @@ -211,6 +212,9 @@ def run(url: str, paths: tuple[str, ...], from_sitemap: bool, max_users: int, journeys, url, config, latency_wall=latency_wall, error_threshold=error_threshold, timeout=timeout, headers=headers, replica=bool(app), as_json=as_json, combined=not no_combined) + except LoadError as exc: + (Console(stderr=True) if as_json else console).print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) except NoJourneyReplaysError: (Console(stderr=True) if as_json else console).print( "[red]Error:[/red] no journey replays against this app, so there's nothing " @@ -386,8 +390,8 @@ def run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wal """The isolation pass: check each journey once, then ramp each one that replays on its own; then, with two or more, the combined pass. Returns (report, result, live_mode) for the first journey to break, with every - journey's verdict in result["journeys"]. Raises NoJourneyReplaysError if every - journey is stale or unreachable. Shared by `run` and `ci`.""" + journey's verdict in result["journeys"]. Raises NoJourneyReplaysError if no + journey replays, and LoadError if a ramp can't run. Shared by `run` and `ci`.""" if not as_json: console.print(f"\n[bold]PreScale[/bold] — replaying {len(journeys)} " f"journey{'s' if len(journeys) != 1 else ''} against [cyan]{url}[/cyan]") @@ -401,7 +405,7 @@ def run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wal name = escape(run.journey.name) if not run.check.ok: failed = run.check.failed - what = "stale" if run.check.stale else "couldn't run" + what = LABELS[run.check.outcome] out.print(f"[yellow]⚠ {name}: {what}, not load-tested[/yellow] " f"[dim]({escape(failed.step.name)}: {escape(failed.problem or '')})" "[/dim]", highlight=False) @@ -411,16 +415,12 @@ def run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wal f"{len(run.journey.steps)} steps)[/dim]", highlight=False) progress = ramp_progress(console, latency_wall=latency_wall, as_json=as_json) live_mode = isinstance(progress, LiveRamp) - try: - with progress: - stages, run.warning = asyncio.run(ramp( - run.journey, url, levels=levels, stage_seconds=config["stage_seconds"], - timeout=timeout, max_rps=config["max_rps"], warmup=config["warmup"], - repeat=config["repeat"], headers=headers, think=config["think_time_s"], - progress_cb=progress.starting, on_stage=progress.finished)) - except LoadError as exc: - out.print(f"[red]Error:[/red] {exc}") - raise SystemExit(1) + with progress: + stages, run.warning = asyncio.run(ramp( + run.journey, url, levels=levels, stage_seconds=config["stage_seconds"], + timeout=timeout, max_rps=config["max_rps"], warmup=config["warmup"], + repeat=config["repeat"], headers=headers, think=config["think_time_s"], + progress_cb=progress.starting, on_stage=progress.finished)) run.report = analyze(stages, latency_wall=latency_wall, error_threshold=error_threshold, rate_capped=config["max_rps"] is not None) @@ -446,17 +446,13 @@ def run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wal highlight=False) progress = ramp_progress(console, latency_wall=latency_wall, as_json=as_json) live_mode = isinstance(progress, LiveRamp) - try: - with progress: - stages, together_warning = asyncio.run(ramp( - [r.journey for r in ramped], url, levels=levels, - stage_seconds=config["stage_seconds"], timeout=timeout, - max_rps=config["max_rps"], warmup=config["warmup"], - repeat=config["repeat"], headers=headers, think=config["think_time_s"], - progress_cb=progress.starting, on_stage=progress.finished)) - except LoadError as exc: - out.print(f"[red]Error:[/red] {exc}") - raise SystemExit(1) + with progress: + stages, together_warning = asyncio.run(ramp( + [r.journey for r in ramped], url, levels=levels, + stage_seconds=config["stage_seconds"], timeout=timeout, + max_rps=config["max_rps"], warmup=config["warmup"], + repeat=config["repeat"], headers=headers, think=config["think_time_s"], + progress_cb=progress.starting, on_stage=progress.finished)) together = analyze(stages, latency_wall=latency_wall, error_threshold=error_threshold, rate_capped=config["max_rps"] is not None) result["combined"] = combined_block(together, runs) diff --git a/cli/src/prescale_cli/journeys/load.py b/cli/src/prescale_cli/journeys/load.py index bd03a9e..ae8920e 100644 --- a/cli/src/prescale_cli/journeys/load.py +++ b/cli/src/prescale_cli/journeys/load.py @@ -27,6 +27,9 @@ ) from prescale_cli.loadtest import RunReport, StageResult, run_loadtest +# How each non-ok outcome reads in reports. +LABELS = {"stale": "stale", "error": "5xx with one user", "unreachable": "couldn't run"} + class NoJourneyReplaysError(Exception): """Every journey was stale or unreachable, so nothing could be load-tested.""" @@ -47,7 +50,7 @@ class JourneyRun: def status(self) -> str: if self.report is not None: return "ok" - return "stale" if self.check.stale else "unreachable" + return self.check.outcome async def ramp(journey: Journey | list[Journey], base_url: str, *, levels: list[int], diff --git a/cli/src/prescale_cli/journeys/model.py b/cli/src/prescale_cli/journeys/model.py index 699133f..e98fc51 100644 --- a/cli/src/prescale_cli/journeys/model.py +++ b/cli/src/prescale_cli/journeys/model.py @@ -53,7 +53,7 @@ _PLACEHOLDER = re.compile(r"\{\{\s*([A-Za-z_][\w.]*)\s*\}\}") # A key is secret-looking when it is, or ends with, one of these words: # password, client_secret, x-api-key, access_token, Authorization, Cookie. -_SENSITIVE_KEY = re.compile( +SENSITIVE_KEY = re.compile( r"(^|[-_.])(password|passwd|pwd|secret|token|api[-_]?key|authorization|cookie)$", re.IGNORECASE) # The secret itself must be a placeholder, optionally after an auth scheme. @@ -279,7 +279,7 @@ def _check_secrets(value: Any, where: str, key: str = "") -> None: elif isinstance(value, list): for v in value: _check_secrets(v, where, key) - elif value is not None and key and _SENSITIVE_KEY.search(key) \ + elif value is not None and key and SENSITIVE_KEY.search(key) \ and not (isinstance(value, str) and _SECRET_OK.fullmatch(value.strip())): raise JourneyError(f"{where}: {key} has a literal value. Put secrets in an " f"environment variable and write \"{{{{env.NAME}}}}\" instead") diff --git a/cli/src/prescale_cli/journeys/replay.py b/cli/src/prescale_cli/journeys/replay.py index 633f226..d6e4f7b 100644 --- a/cli/src/prescale_cli/journeys/replay.py +++ b/cli/src/prescale_cli/journeys/replay.py @@ -25,7 +25,14 @@ import httpx -from prescale_cli.journeys.model import Journey, Step, TemplateError, fresh_uniques, render +from prescale_cli.journeys.model import ( + SENSITIVE_KEY, + Journey, + Step, + TemplateError, + fresh_uniques, + render, +) from prescale_cli.loadtest import client_headers _SNIPPET = 160 @@ -76,8 +83,11 @@ class StepCheck: status: int | None = None latency: float = 0.0 ok: bool = False - problem: str | None = None # why it failed, in plain words + problem: str | None = None # why it failed, in plain words (no response body) kind: str | None = None # a short failure kind for load reports + # The start of the response body, secrets masked. Shown in a local terminal + # only: never in CI output, a PR comment or an uploaded Result. + snippet: str | None = None headers: httpx.Headers | None = None @@ -95,10 +105,20 @@ def failed(self) -> StepCheck | None: return next((s for s in self.steps if not s.ok), None) @property - def stale(self) -> bool: - """The app answered, but not the way the journey expects.""" + def outcome(self) -> str: + """ok; stale (the app answered, but not as the journey expects, e.g. a + 404: the app changed); error (a 5xx with a single user: the app is + broken); unreachable (no answer at all).""" failed = self.failed - return failed is not None and failed.status is not None + if failed is None and self.ok: + return "ok" + if failed is None or failed.status is None: + return "unreachable" + return "error" if failed.status >= 500 else "stale" + + @property + def stale(self) -> bool: + return self.outcome == "stale" async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, @@ -108,12 +128,13 @@ async def check(journey: Journey, base_url: str, *, timeout: float = 10.0, env = dict(os.environ) if env is None else env result = JourneyCheck(journey) variables: dict[str, Any] = fresh_uniques() + secrets = secret_values(journey, env) jar = httpx.Cookies() async with httpx.AsyncClient(timeout=timeout, follow_redirects=True, headers=client_headers(headers), transport=transport, **client_options(base_url)) as client: for step in journey.steps: - outcome = await run_step(client, step, base_url, variables, env, jar) + outcome = await run_step(client, step, base_url, variables, env, jar, secrets) result.steps.append(outcome) if not outcome.ok: break @@ -195,9 +216,25 @@ def _record(sink, outcome: StepCheck, prefix: str = "") -> None: sink.record_error(label, outcome.kind or f"unexpected {outcome.status}") +def secret_values(journey: Journey, env: dict[str, str]) -> set[str]: + """The secrets a journey sends: the environment variables it reads.""" + return {env[name] for name in journey.env_names() if env.get(name)} + + +def _mask(text: str, secrets: set[str], variables: dict[str, Any]) -> str: + """TEXT with every secret replaced by ***: the journey's env values, and any + extracted value whose name looks secret (a token, a session id).""" + hidden = set(secrets) | {str(v) for k, v in variables.items() + if SENSITIVE_KEY.search(k) and isinstance(v, str)} + for value in sorted(hidden, key=len, reverse=True): + if len(value) >= 4: + text = text.replace(value, "***") + return text + + async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, variables: dict[str, Any], env: dict[str, str], - jar: httpx.Cookies) -> StepCheck: + jar: httpx.Cookies, secrets: set[str] | None = None) -> StepCheck: """Send one step with this journey run's cookies (JAR), judge it against `expect`, and pull its `extract` values into VARIABLES for the steps after it.""" outcome = StepCheck(step) @@ -236,9 +273,9 @@ async def run_step(client: httpx.AsyncClient, step: Step, base_url: str, if not step.accepts(resp.status_code): wanted = (" or ".join(map(str, step.expect)) if step.expect else "a status below 400") outcome.problem = f"returned {resp.status_code}, expected {wanted}" - snippet = resp.text.strip().replace("\n", " ")[:_SNIPPET] - if snippet: - outcome.problem += f": {snippet}" + # Bodies can echo what was sent (validation errors often do), secrets included. + snippet = _mask(resp.text.strip().replace("\n", " "), secrets or set(), variables) + outcome.snippet = snippet[:_SNIPPET] or None return outcome for name, source in step.extract.items(): diff --git a/cli/src/prescale_cli/render.py b/cli/src/prescale_cli/render.py index fe5e821..ef37f61 100644 --- a/cli/src/prescale_cli/render.py +++ b/cli/src/prescale_cli/render.py @@ -9,6 +9,7 @@ from rich.table import Table from prescale_cli.bars import text_bar +from prescale_cli.journeys.load import LABELS from prescale_cli.throughput import VISITOR_NOTE, describe console = Console() @@ -160,7 +161,8 @@ def _expect_lines(exp: dict) -> list[str]: mark = {True: "✅", False: "🛑", None: "❔"}[c["meets"]] lines.append(f" {mark} all together: holds ~{c['survives_users']}") for name in exp.get("untested") or []: - lines.append(f" ❔ {escape(name)}: not load-tested (stale or unreachable)") + lines.append(f" ❔ {escape(name)}: not load-tested " + "(stale, failing or unreachable)") return lines @@ -176,7 +178,7 @@ def render_journeys(result: dict) -> None: for j in result["journeys"]: name = escape(j["name"]) if j["status"] != "ok": - label = "stale" if j["status"] == "stale" else "couldn't run" + label = LABELS.get(j["status"], j["status"]) table.add_row(name, f"[yellow]{label}[/yellow]", "—", f"[dim]{escape(j.get('reason') or '')}[/dim]") continue diff --git a/cli/src/prescale_cli/report.py b/cli/src/prescale_cli/report.py index 07720b6..2ea2d0a 100644 --- a/cli/src/prescale_cli/report.py +++ b/cli/src/prescale_cli/report.py @@ -10,6 +10,8 @@ import html from datetime import datetime +from prescale_cli.journeys.load import LABELS + _CSS = """ :root{ --canvas:#010102;--s1:#0f1011;--s2:#141516;--hair:#23252a; @@ -275,7 +277,7 @@ def _journeys_html(result: dict) -> str: rows = [] for j in journeys: if j["status"] != "ok": - label = "stale" if j["status"] == "stale" else "couldn't run" + label = LABELS.get(j["status"], j["status"]) rows.append(f"{_esc(j['name'])}{label}" f"—" f"{_esc(j.get('reason') or '')}") @@ -334,7 +336,7 @@ def _expect_html(exp: dict | None) -> str: if combined: items.append(f"
  • all together: holds ~{combined['survives_users']}
    • ") for name in exp.get("untested") or []: - items.append(f"
  • {_esc(name)}: not load-tested (stale or unreachable)
    • ") + items.append(f"
  • {_esc(name)}: not load-tested (stale, failing or unreachable)
    • ") if not exp.get("journeys"): items.append(f"
  • survives ~{exp['survives_users']}
    • ") return (f'
    Expected ~{exp["users"]} users: {answer}' diff --git a/cli/src/prescale_cli/schema/result.schema.json b/cli/src/prescale_cli/schema/result.schema.json index 7751c93..622f890 100644 --- a/cli/src/prescale_cli/schema/result.schema.json +++ b/cli/src/prescale_cli/schema/result.schema.json @@ -182,7 +182,7 @@ "name": { "type": "string" }, "type": { "type": "string" }, "weight": { "type": "number" }, - "status": { "enum": ["ok", "stale", "unreachable"] }, + "status": { "enum": ["ok", "stale", "error", "unreachable"], "description": "stale: answered, but not as the journey expects (the app changed); error: a 5xx with one user; unreachable: no answer." }, "reason": { "type": ["string", "null"], "description": "Why a stale or unreachable journey wasn't load-tested." }, "survives_users": { "type": "integer" }, "max_tested": { "type": "integer" }, diff --git a/cli/tests/test_action.py b/cli/tests/test_action.py index c44682d..6fcf9a5 100644 --- a/cli/tests/test_action.py +++ b/cli/tests/test_action.py @@ -32,6 +32,14 @@ def test_capacity_check_passes_stale_and_each_allow_writes_host(tmp_path): "INPUT_HOST_PERMISSION": "false", "INPUT_APP": "", "INPUT_APP_TIMEOUT": "600", "INPUT_URL": "", "INPUT_STALE": "fail", "INPUT_ALLOW_WRITES": "staging.myapp.com\n qa.myapp.com \n\n"} - out = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True).stdout - assert out.endswith("[--stale][fail][--allow-writes][staging.myapp.com]" - "[--allow-writes][qa.myapp.com]") + def run(**inputs): + return subprocess.run(["bash", "-eo", "pipefail", "-c", script], env={**env, **inputs}, + capture_output=True, text=True).stdout + + assert run().endswith("[--stale][fail][--allow-writes][staging.myapp.com]" + "[--allow-writes][qa.myapp.com]") + # the default passes no new flag, so a pinned older prescale-version keeps working + assert "--stale" not in run(INPUT_STALE="warn", INPUT_ALLOW_WRITES="") + assert "--stale" not in run(INPUT_STALE="", INPUT_ALLOW_WRITES="") + assert run(INPUT_STALE="Fail", INPUT_ALLOW_WRITES="a.com, b.com").endswith( + "[--stale][fail][--allow-writes][a.com][--allow-writes][b.com]") diff --git a/cli/tests/test_ci_journeys.py b/cli/tests/test_ci_journeys.py index 95d285f..9e00df3 100644 --- a/cli/tests/test_ci_journeys.py +++ b/cli/tests/test_ci_journeys.py @@ -189,3 +189,59 @@ def test_ci_refuses_journey_writes_on_a_preview_host(monkeypatch, tmp_path): "--dir", _dir(tmp_path)]) assert res.exit_code == 2 assert "--allow-writes preview.example.com" in " ".join(res.output.split()) + + +# --- review fixes -------------------------------------------------------------------- + +def _runs(outcomes): + runs = [] + for name, status in outcomes: + j = parse({**BROWSE, "name": name}) + c = JourneyCheck(j) + c.steps = [StepCheck(j.steps[0], status=status, problem=f"returned {status}" + if status else "couldn't reach the app (ConnectError)")] + runs.append(JourneyRun(j, c)) + return runs + + +def test_an_app_that_doesnt_answer_fails_the_check_even_with_stale_warn(monkeypatch, tmp_path): + _wire(monkeypatch, raises=NoJourneyReplaysError(_runs([("a", None), ("b", 404)]))) + res = _ci(tmp_path) + assert res.exit_code == 1 + assert "::error::PreScale: a failed with a single user" in res.output + + +def test_a_5xx_with_one_user_fails_the_check(monkeypatch, tmp_path): + _wire(monkeypatch, raises=NoJourneyReplaysError(_runs([("a", 500)]))) + assert _ci(tmp_path).exit_code == 1 + result = _journey_result() + result["journeys"][1] = {"name": "browse", "type": "browse", "weight": 60, + "status": "error", "reason": "step 1 (home): returned 500"} + _wire(monkeypatch, result=result) + res = _ci(tmp_path) + assert res.exit_code == 1 + assert "::error::PreScale: browse failed with a single user" in res.output + + +def test_only_stale_follows_the_stale_policy(monkeypatch, tmp_path): + _wire(monkeypatch, raises=NoJourneyReplaysError(_runs([("a", 404)]))) + assert _ci(tmp_path).exit_code == 0 + + +def test_comment_text_from_files_and_apps_cant_mention_render_html_or_break_code(): + result = _journey_result(stale=True) + result["journeys"][1]["reason"] = "returned 422 for @octocat | ok" + result["journeys"][0]["culprit_step"]["route"] = "POST /a`b" + body = ci_mod.render_comment(result, _upload(mode="journeys"), target="t", sha=None) + assert "@octocat" not in body and "@​octocat" in body + assert " Date: Tue, 3 Nov 2026 15:12:54 +0000 Subject: [PATCH 11/11] test: wrap two long lines in the journey tests --- cli/tests/test_ci_journeys.py | 3 ++- cli/tests/test_journeys.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/tests/test_ci_journeys.py b/cli/tests/test_ci_journeys.py index 9e00df3..f48e46d 100644 --- a/cli/tests/test_ci_journeys.py +++ b/cli/tests/test_ci_journeys.py @@ -12,7 +12,8 @@ from prescale_cli.main import cli CHECKOUT = {"version": 1, "name": "checkout", "type": "checkout", - "steps": [{"name": "place order", "request": {"method": "POST", "path": "/api/orders"}}]} + "steps": [{"name": "place order", + "request": {"method": "POST", "path": "/api/orders"}}]} BROWSE = {"version": 1, "name": "browse", "steps": [{"name": "home", "request": {"path": "/"}}]} diff --git a/cli/tests/test_journeys.py b/cli/tests/test_journeys.py index 60cc0bf..e8aef66 100644 --- a/cli/tests/test_journeys.py +++ b/cli/tests/test_journeys.py @@ -273,7 +273,8 @@ def handler(request): return httpx.Response(200) j = _one_step(path="/{{next}}") - result = asyncio.run(check(j, "http://shop.test", env={}, transport=httpx.MockTransport(handler))) + result = asyncio.run(check(j, "http://shop.test", env={}, + transport=httpx.MockTransport(handler))) assert seen == ["http://shop.test/start"] # nothing sent off the app assert not result.ok and "single /" in result.failed.problem