diff --git a/CHANGELOG.md b/CHANGELOG.md index df9e7f2..d31983c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,16 @@ pinned older `prescale-version` keeps working). Until the dashboard compares journey runs only with journey baselines, a journey run's PR comment shows no change instead of a possibly misleading one, and never fails on it. +- **`investigate` diagnoses the journey step that breaks first.** With journeys in + `prescale/journeys/`, `prescale investigate` ramps them like `run`, then probes the step that + broke first with the journey's own state (its 1-user latency comes from replaying the journey + from the start), compares it with the journey's other steps, and names the bottleneck and the + fix: `culprit checkout, step 3 place order · connection_pool (high confidence)`. If the + journeys ramped together starved that journey, the evidence says so; if every journey held alone + but they broke together, the step that broke together is diagnosed. A step that answers an + unexpected 4xx under load gets a new class, `unexpected_answer` (users conflicting over shared + data, or sessions failing under load). The `investigation` field gains `culprit_journey`, + `culprit_step` and `interference`. `--urls` investigates URLs as before. - **`--expect N`** on `run` says whether you'd hold N users at peak. In a journey run, N is split across journeys by weight: each must hold its share alone, and all together must hold N. diff --git a/README.md b/README.md index 1515994..cc9767c 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,8 @@ Try this → Add a pooler (e.g. pgbouncer) and check the pool checkout timeout. ``` +With journeys in `prescale/journeys/`, `investigate` diagnoses the journey step that breaks first instead, reaching it the way a user does (logged in, cart filled): `culprit checkout, step 3 place order`. The journey's other steps play the part of the static assets. `--urls` investigates URLs. + ### `journeys` — your users' real paths A journey is a path a user takes through your app (log in, add to cart, place an order), saved as the requests it makes. Journeys live in your repo as `prescale/journeys/.yaml`: diff --git a/cli/pyproject.toml b/cli/pyproject.toml index bf467a3..1b7382b 100644 --- a/cli/pyproject.toml +++ b/cli/pyproject.toml @@ -40,6 +40,7 @@ mcp = [ dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", + "jsonschema>=4.0", ] [project.scripts] diff --git a/cli/src/prescale_cli/commands/investigate.py b/cli/src/prescale_cli/commands/investigate.py index 0d5ed6a..11ac23a 100644 --- a/cli/src/prescale_cli/commands/investigate.py +++ b/cli/src/prescale_cli/commands/investigate.py @@ -4,6 +4,7 @@ import asyncio import json +from pathlib import Path from urllib.parse import urlparse import click @@ -13,15 +14,18 @@ from prescale_cli.cloud import CloudError, require_credentials from prescale_cli.commands.audit import _ICONS as _AUDIT_ICONS from prescale_cli.commands.hostcheck import confirm_host, host_permission_option +from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica +from prescale_cli.commands.run import _journeys_to_run, run_journeys from prescale_cli.commands.upload import header_option, upload_after_run, upload_option -from prescale_cli.investigate import _LATENCY_WALL_S +from prescale_cli.investigate import _ERROR_THRESHOLD, _LATENCY_WALL_S, audit_block from prescale_cli.investigate import investigate as run_investigate +from prescale_cli.journeys.load import NoJourneyReplaysError from prescale_cli.live import LiveRamp, ramp_progress from prescale_cli.loadtest import LoadError -from prescale_cli.profiles import PROFILES, lookup +from prescale_cli.profiles import PROFILES, lookup, scenario_block from prescale_cli.render import render_investigation, render_terminal -from prescale_cli.result import latest_id, load_result +from prescale_cli.result import latest_id, load_result, write_result console = Console() @@ -47,7 +51,18 @@ help="Test against a launch scenario (see `prescale profiles`); ramps to 1.5x " "its peak to show headroom.") @click.option("--audit", "with_audit", is_flag=True, - help="Also run the load-free HTTP hygiene audit (what Launch Guard shows).") + help="Also run the load-free HTTP hygiene audit.") +@click.option("--journey", "only", multiple=True, + help="Investigate only this journey (repeatable). Default: every journey " + "in prescale/journeys/.") +@click.option("--no-combined", "no_combined", is_flag=True, + help="Skip ramping the journeys together (no interference check).") +@click.option("--urls", "urls_only", is_flag=True, + help="Investigate URLs, even when prescale/journeys/ has journeys.") +@click.option("--allow-writes", "allow_writes", multiple=True, metavar="HOST", + help="Let journeys that change data run against this non-local host, " + "e.g. staging.myapp.com (repeatable).") +@dir_option @app_option @app_timeout_option @host_permission_option @@ -56,12 +71,16 @@ def investigate(url: str | None, paths: tuple[str, ...], max_users: int, stage_seconds: float, max_rps: float | None, yes: bool, as_json: bool, store: str | None, fail_under: int | None, - profile_name: str | None, with_audit: bool, app: str | None, - app_timeout: float, host_permission: bool, headers: dict[str, str], - upload: bool) -> None: + profile_name: str | None, with_audit: bool, only: tuple[str, ...], + no_combined: bool, urls_only: bool, allow_writes: tuple[str, ...], + directory: Path, app: str | None, app_timeout: float, + host_permission: bool, headers: dict[str, str], upload: bool) -> None: """Find what breaks first, then probe it to explain WHY - and how to fix it. - With no URL, re-investigates the latest saved run's target. + When prescale/journeys/ has journeys, the journey and step that break first + are probed (with the journey's state, e.g. logged in, cart filled). Use + --urls to investigate URLs instead. With no URL, re-investigates the latest + saved run's target. \b Examples: @@ -86,7 +105,8 @@ def investigate(url: str | None, paths: tuple[str, ...], max_users: int, "[bold]prescale investigate [/bold].[/dim]") raise SystemExit(0) url = load_result(last, store=store)["target"]["url"] - console.print(f"[dim]Re-investigating latest run: {url}[/dim]") + (Console(stderr=True) if as_json else console).print( + f"[dim]Re-investigating latest run: {url}[/dim]") prof = lookup(profile_name) if profile_name else None if profile_name and prof is None: @@ -103,6 +123,12 @@ def investigate(url: str | None, paths: tuple[str, ...], max_users: int, "(expected e.g. http://localhost:8000).") raise SystemExit(1) + journeys = _journeys_to_run(directory, only, urls_only=urls_only, url_mode=bool(paths)) + if journeys: + host = (parsed.hostname or "").lower() + refuse_unallowed_writes(journeys, host, {h.lower() for h in allow_writes}) + refuse_missing_env(journeys) + if app: start_replica(app, url, timeout=app_timeout, console=console, quiet=as_json) @@ -123,6 +149,15 @@ def investigate(url: str | None, paths: tuple[str, ...], max_users: int, confirm_host(url, console=console, permitted=host_permission, as_json=as_json, headers=headers) + if journeys: + result, live_mode = _investigate_journeys( + journeys, url, max_users=max_users, stage_seconds=stage_seconds, + max_rps=max_rps, prof=prof, headers=headers, replica=bool(app), + as_json=as_json, combined=not no_combined, with_audit=with_audit, store=store) + _show(result, as_json=as_json, live_mode=live_mode, upload=upload, + fail_under=fail_under) + return + progress = ramp_progress(console, latency_wall=_LATENCY_WALL_S, as_json=as_json) live_mode = isinstance(progress, LiveRamp) try: @@ -137,7 +172,43 @@ def investigate(url: str | None, paths: tuple[str, ...], max_users: int, except LoadError as exc: console.print(f"[red]Error:[/red] {exc}") raise SystemExit(1) + _show(result, as_json=as_json, live_mode=live_mode, upload=upload, fail_under=fail_under) + + +def _investigate_journeys(journeys, url: str, *, max_users: int, stage_seconds: float, + max_rps: float | None, prof, headers: dict[str, str], + replica: bool, as_json: bool, combined: bool, with_audit: bool, + store: str | None) -> tuple[dict, bool]: + """Ramp the journeys (as `run` does), then probe the step that broke first.""" + config = {"method": "GET", "max_users": max_users, "stage_seconds": stage_seconds, + "latency_wall_s": _LATENCY_WALL_S, "error_threshold": _ERROR_THRESHOLD, + "max_rps": max_rps, "warmup": True, "repeat": 1, + "think_time_s": prof.think_time_s if prof else 0.0, + "profile": prof.name if prof else None} + err = Console(stderr=True) if as_json else console + try: + report, result, live_mode = run_journeys( + journeys, url, config, latency_wall=_LATENCY_WALL_S, + error_threshold=_ERROR_THRESHOLD, timeout=10.0, headers=headers, + replica=replica, as_json=as_json, combined=combined, diagnose=True) + except LoadError as exc: + err.print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) + except NoJourneyReplaysError: + err.print("[red]Error:[/red] no journey replays against this app, so there's " + "nothing to investigate. Run `prescale journeys verify` for details.") + raise SystemExit(1) + if prof is not None: + result["profile"] = scenario_block(prof, report.survives_users) + if with_audit: + result["audit"] = asyncio.run(audit_block(url, headers=headers)) + write_result(result, store=store) + return result, live_mode + +def _show(result: dict, *, as_json: bool, live_mode: bool, upload: bool, + fail_under: int | None) -> None: + """Print, upload and gate an investigated Result (URLs or journeys alike).""" if as_json: click.echo(json.dumps(result, indent=2)) else: @@ -149,13 +220,25 @@ def investigate(url: str | None, paths: tuple[str, ...], max_users: int, icon = _AUDIT_ICONS.get(finding["status"], "•") console.print(f"{icon} [bold]{finding['name']}[/bold] {finding['detail']}") if result.get("investigation") is None: - console.print("\n[green]✓[/green] Held up — nothing to diagnose.") + if (result.get("combined") or {}).get("onset_users") is not None \ + or result["verdict"]["onset_users"] is not None: + console.print("\n[yellow]⚠[/yellow] It broke, but PreScale couldn't probe the " + "step that broke (see above).") + else: + console.print("\n[green]✓[/green] Held up — nothing to diagnose.") console.print(f"[dim]saved as {result['id']}[/dim]") if upload: upload_after_run(result, as_json=as_json) survives = result["verdict"]["survives_users"] + together = result.get("combined") or {} + if together.get("onset_users") is not None and together["survives_users"] < survives: + # The journeys broke sooner together than any did alone: that's the number to gate on. + survives = together["survives_users"] + if not as_json: + console.print(f"[dim]Together, the journeys survive ~{survives} users in total; " + "--fail-under uses that.[/dim]") if fail_under is not None and survives < fail_under: Console(stderr=True).print( f"[red]✗ fail-under:[/red] survives ~{survives} < {fail_under}") diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index d3da268..8edb95f 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -21,6 +21,7 @@ from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica from prescale_cli.commands.upload import header_option, upload_after_run, upload_option +from prescale_cli.journeys.diagnose import diagnose_run from prescale_cli.journeys.load import ( LABELS, JourneyRun, @@ -386,12 +387,15 @@ def _journeys_to_run(directory: Path, only: tuple[str, ...], *, urls_only: bool, def run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wall: float, error_threshold: float, timeout: float, headers: dict[str, str], - replica: bool, as_json: bool, combined: bool = True) -> tuple: + replica: bool, as_json: bool, combined: bool = True, + diagnose: bool = False) -> tuple: """The isolation pass: check each journey once, then ramp each one that replays on its own; then, with two or more, the combined pass. Returns (report, result, live_mode) for the first journey to break, with every - journey's verdict in result["journeys"]. Raises NoJourneyReplaysError if no - journey replays, and LoadError if a ramp can't run. Shared by `run` and `ci`.""" + journey's verdict in result["journeys"]. With DIAGNOSE (`investigate`), the + step that broke first is probed and result["investigation"] says why. Raises + NoJourneyReplaysError if no journey replays, and LoadError if a ramp can't + run. Shared by `run`, `investigate` and `ci`.""" if not as_json: console.print(f"\n[bold]PreScale[/bold] — replaying {len(journeys)} " f"journey{'s' if len(journeys) != 1 else ''} against [cyan]{url}[/cyan]") @@ -439,6 +443,7 @@ def run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wal result["journeys"] = journeys_block(runs) ramped = [r for r in runs if r.report is not None] + together = None if combined and len(ramped) > 1: if not as_json: mix = ", ".join(f"{escape(r.journey.name)} {r.journey.weight:g}" for r in ramped) @@ -458,6 +463,17 @@ def run_journeys(journeys: list[Journey], url: str, config: dict, *, latency_wal result["combined"] = combined_block(together, runs) if together_warning and together_warning not in (result["warning"] or ""): result["warning"] = " ".join(filter(None, [result["warning"], together_warning])) + + broke_together = together is not None and together.onset_users is not None + if diagnose and (worst.report.onset_users is not None or broke_together): + if not as_json: + console.print("\n[dim]Diagnosing the step that broke first…[/dim]") + investigation = asyncio.run(diagnose_run( + worst, runs, url, together=together, combined=result.get("combined"), + latency_wall=latency_wall, error_threshold=error_threshold, timeout=timeout, + headers=headers, max_rps=config["max_rps"])) + if investigation is not None: + result["investigation"] = investigation return worst.report, result, live_mode diff --git a/cli/src/prescale_cli/investigate.py b/cli/src/prescale_cli/investigate.py index 1ab06be..f3c1655 100644 --- a/cli/src/prescale_cli/investigate.py +++ b/cli/src/prescale_cli/investigate.py @@ -51,6 +51,14 @@ class Probes: # Share of the culprit's successful responses a CDN served from cache, and which CDN. cache_share: float | None = None cache_source: str | None = None + # What static_ok compares against: static assets for URL routes; for a journey + # step, the journey's other steps (they held while this one broke, or not). + reference: str = "static assets" + # The 1-user figure ("p95", or "median" when there are too few samples for a + # p95) and what the onset level counts ("users", or "users in total" when + # several journeys ran together). + baseline_stat: str = "p95" + users_label: str = "users" @dataclass @@ -67,12 +75,19 @@ def _capitalize(text: str) -> str: def _evidence(p: Probes) -> list[str]: ev: list[str] = [] + users = f"{p.onset_users} {p.users_label}" if p.baseline_p95_ms is not None and p.loaded_p95_ms is not None: - ev.append(f"culprit p95 {p.baseline_p95_ms:.0f}ms at 1 user vs " - f"{p.loaded_p95_ms:.0f}ms at {p.onset_users} users") + if p.baseline_stat == "p95": + ev.append(f"culprit p95 {p.baseline_p95_ms:.0f}ms at 1 user vs " + f"{p.loaded_p95_ms:.0f}ms at {users}") + else: + ev.append(f"culprit {p.baseline_stat} {p.baseline_p95_ms:.0f}ms at 1 user vs " + f"p95 {p.loaded_p95_ms:.0f}ms at {users}") if p.static_route is not None and p.static_ok is not None: held = "held" if p.static_ok else "also degraded" - ev.append(f"static {p.static_route} {held} at {p.onset_users} users") + what = (f"static {p.static_route}" if p.reference == "static assets" + else f"{p.reference} ({p.static_route})") + ev.append(f"{what} {held} at {users}") if p.error_kinds: worst = sorted(p.error_kinds.items(), key=lambda kv: -kv[1]) ev.append("errors under load: " + ", ".join(f"{k} ({v})" for k, v in worst)) @@ -121,15 +136,23 @@ def classify(p: Probes) -> Diagnosis: if dominant == "timeout": if p.static_ok is False: return Diagnosis("connection_ceiling", "medium", - "Timeouts with static assets degrading too — saturated at " + f"Timeouts with {p.reference} degrading too — saturated at " "the connection level.", ev) return Diagnosis("connection_pool", "medium", "Requests timed out while infra held — a backend pool or " "downstream dependency is blocking.", ev) + if dominant and (dominant.startswith("unexpected 4") or dominant == "missing extract"): + # Journey steps only: it works for one user, but under load it answered + # something the journey doesn't expect. + return Diagnosis("unexpected_answer", "medium", + f"Under load the step answered something it doesn't for one user " + f"({dominant}) — often users conflicting over shared data (a 409, a " + "lock, a unique constraint) or sessions and tokens that don't hold " + "up under load (401/403).", ev) if has_other_5xx: if p.static_ok is True: return Diagnosis("connection_pool", "high", - "5xx under load while static assets held — the app/backend " + f"5xx under load while {p.reference} held — the app/backend " "is the wall (often a DB or upstream pool).", ev) return Diagnosis("connection_pool", "medium", "5xx under load — an unhandled overload in the app " @@ -190,6 +213,13 @@ def classify(p: Probes) -> Diagnosis: "Profile the route — it's slow even at 1 user (likely a slow query or N+1).", "Add an index, a cache, or pagination; move heavy work off the request path.", ], + "unexpected_answer": [ + "Check the app's logs for this step under load: a 409 or a unique-constraint error " + "points at rows several users change at once (lock them, or retry); a 401/403 at a " + "session store or token cache that doesn't keep up.", + "Make sure each virtual user's data is its own: a journey where every user changes " + "the same account or item can conflict by design.", + ], "unknown": [ "Re-run with --repeat 3 for a tighter read, and inspect the per-route breakdown.", "Check app logs and DB/connection metrics around the onset level.", @@ -322,14 +352,18 @@ async def investigate(url, *, max_users=200, paths=(), stage_seconds=5.0, result["profile"] = scenario_block(profile, report.survives_users) if audit: - # Load-free hygiene checks, so Launch Guard has the whole picture in one run. - try: - findings = await run_audit(url, extra_headers=headers) - result["audit"] = [{"name": f.name, "status": f.status, "detail": f.detail, - "fix": f.fix} for f in findings] - except AuditError as exc: - result["audit"] = [{"name": "Audit", "status": "info", - "detail": f"Couldn't run the audit: {exc}", "fix": None}] + result["audit"] = await audit_block(url, headers=headers) write_result(result, store=store) return result + + +async def audit_block(url: str, *, headers=None) -> list[dict]: + """The Result's `audit` field: load-free HTTP hygiene checks.""" + try: + findings = await run_audit(url, extra_headers=headers) + return [{"name": f.name, "status": f.status, "detail": f.detail, "fix": f.fix} + for f in findings] + except AuditError as exc: + return [{"name": "Audit", "status": "info", + "detail": f"Couldn't run the audit: {exc}", "fix": None}] diff --git a/cli/src/prescale_cli/journeys/diagnose.py b/cli/src/prescale_cli/journeys/diagnose.py new file mode 100644 index 0000000..e6156eb --- /dev/null +++ b/cli/src/prescale_cli/journeys/diagnose.py @@ -0,0 +1,234 @@ +"""Why the journey step that breaks first broke, and how to fix it. + +The ramp names the journey and the step that break first. This probes that step +the way `investigate` probes a URL route, but with the journey's own state: its +latency with one user comes from replaying the journey from the start (log in, +add to cart, then the step), so a step that needs a token or a cart id is +measured the way a real user reaches it. The journey's other steps play the part +static assets play for URLs: if they held while this step broke, the limit is +something this step uses (often a database pool), not the whole server. + +When every journey held on its own but they broke ramped together, the step that +broke in the combined run is diagnosed instead, from the combined run's numbers: +that's the case where journeys compete for something they share. + +Classification and fixes are `investigate`'s (`classify`, `remediate`), so a +journey step and a URL route get the same diagnosis for the same evidence. No +model is involved. +""" + +from __future__ import annotations + +import asyncio +import statistics +import time +from dataclasses import dataclass, field + +import httpx + +from prescale_cli.audit import _CDN_MARKERS +from prescale_cli.investigate import Probes, classify, remediate +from prescale_cli.journeys.load import JourneyRun +from prescale_cli.journeys.model import Journey +from prescale_cli.journeys.replay import SEP, check +from prescale_cli.loadtest import RouteStat, RunReport, edge_blocker + +# The 1-user baseline: replay the journey for about this long, at least +# MIN_WALKS and at most MAX_WALKS times, after a short pause so the app can +# recover from the ramp. Fewer than MIN_SAMPLES walks that reach the step and +# there is no baseline (rather than one built on an outlier). +BASELINE_SECONDS = 2.0 +COOLDOWN_SECONDS = 1.0 +MIN_WALKS, MAX_WALKS = 5, 50 +MIN_SAMPLES = 5 +# Below this many samples a p95 is just the slowest one, so use the median. +P95_SAMPLES = 20 + + +@dataclass +class StepProbes: + """Evidence about one journey step: the probes `classify` reads, plus what + only a journey has.""" + probes: Probes + journey: Journey + index: int # 1-based, first step with this route + shared: list[int] = field(default_factory=list) # every step with this route, if >1 + together: bool = False # from the combined run + walks: int = 0 + samples: int = 0 + + +def _held(stat: RouteStat | None, *, latency_wall: float, error_threshold: float) -> bool: + return (stat is not None and stat.total > 0 and stat.error_rate < error_threshold + and (not stat.latencies or stat.pct(0.95) < latency_wall)) + + +def _baseline(samples: list[float]) -> float | None: + if len(samples) < MIN_SAMPLES: + return None + if len(samples) < P95_SAMPLES: + return statistics.median(samples) + ordered = sorted(samples) + return ordered[round(0.95 * (len(ordered) - 1))] + + +async def gather_step_probes(journey: Journey, report: RunReport | None, base_url: str, *, + latency_wall: float, error_threshold: float, prefix: str = "", + timeout: float = 10.0, headers: dict[str, str] | None = None, + env: dict[str, str] | None = None, max_rps: float | None = None, + transport: httpx.AsyncBaseTransport | None = None, + seconds: float = BASELINE_SECONDS, + cooldown: float = COOLDOWN_SECONDS) -> StepProbes | None: + """Evidence about the step of JOURNEY that REPORT broke at, or None if it + didn't break there. PREFIX is how REPORT labels this journey's steps + (`checkout › ` in the combined run).""" + if report is None or report.onset_users is None or not report.culprit_route: + return None + if not report.culprit_route.startswith(prefix): + return None + label = report.culprit_route[len(prefix):] + steps = journey.steps + # Steps with the same method and path share one set of numbers under load. + shared = [i for i, s in enumerate(steps, 1) if s.label == label] + if not shared: + return None + index = shared[0] + onset = report.onset_users + stage = next((s for s in report.stages if s.users == onset), None) + stat = stage.routes.get(prefix + label) if stage else None + p = Probes( + culprit_route=label, onset_users=onset, + loaded_p95_ms=stat.pct(0.95) * 1000 if (stat and stat.latencies) else None, + throughput_plateaued=report.saturated, + error_kinds=dict(stat.error_kinds) if stat else {}, + status_counts=dict(stat.status_counts) if stat else {}, + reference="the journey's other steps", + users_label="users in total" if prefix else "users", + ) + + # The other steps (other routes) that ran at the onset level: steps after a + # failing one don't run in that walk, so only the ones with requests count. + others: dict[str, list[str]] = {} + if stage is not None: + for s in steps: + ran = (stage.routes.get(prefix + s.label) or RouteStat()).total > 0 + if s.label != label and ran: + others.setdefault(s.label, []).append(s.name) + if others: + p.static_route = ", ".join(n for names in others.values() for n in names) + p.static_ok = all(_held(stage.routes.get(prefix + lbl), latency_wall=latency_wall, + error_threshold=error_threshold) for lbl in others) + + # How much of the step a CDN answered from cache, and whether its firewall stepped in. + step_stats = [s.routes[prefix + label] for s in report.stages if prefix + label in s.routes] + served = sum(len(r.latencies) for r in step_stats) + if served: + p.cache_share = sum(r.cache_hits for r in step_stats) / served + p.cache_source = next((r.cache_source for r in step_stats if r.cache_source), None) + p.edge_blocker = next((kind.removeprefix("blocked by ") for r in step_stats + for kind in r.error_kinds if kind.startswith("blocked by ")), None) + + # The step at one user, reached through the journey's earlier steps. + await asyncio.sleep(cooldown) + samples: list[float] = [] + seen_headers: httpx.Headers | None = None + walks = 0 + start = time.monotonic() + while walks < MAX_WALKS and (walks < MIN_WALKS or time.monotonic() - start < seconds): + walk = await check(journey, base_url, timeout=timeout, headers=headers, env=env, + transport=transport) + walks += 1 + if len(walk.steps) >= index and walk.steps[index - 1].ok: + samples.append(walk.steps[index - 1].latency) + seen_headers = walk.steps[index - 1].headers or seen_headers + if max_rps: + # Stay under --max-rps: one walk is one request per step it ran. + await asyncio.sleep(len(walk.steps) / max_rps) + baseline = _baseline(samples) + p.baseline_p95_ms = baseline * 1000 if baseline is not None else None + p.baseline_stat = "p95" if len(samples) >= P95_SAMPLES else "median" + if seen_headers is not None: + lower = {k.lower(): v for k, v in seen_headers.items()} + p.server = lower.get("server") + p.cdn = next((name for marker, name in _CDN_MARKERS.items() if marker in lower), None) + if not p.cdn and "cloudflare" in (p.server or "").lower(): + p.cdn = "Cloudflare" + p.edge_blocker = p.edge_blocker or edge_blocker(seen_headers) + return StepProbes(p, journey, index, shared if len(shared) > 1 else [], + together=bool(prefix), walks=walks, samples=len(samples)) + + +def _names(items: list[str]) -> str: + return items[0] if len(items) == 1 else ", ".join(items[:-1]) + " and " + items[-1] + + +def diagnose_step(found: StepProbes, *, interference: dict | None = None, + held_alone: int | None = None) -> dict: + """The Result's `investigation` block for a journey step: the URL fields + (culprit_route, class, confidence, summary, evidence, remediation, stack), + plus the journey and step, and the combined run's interference when it was + this journey that the mix starved.""" + p = found.probes + diag = classify(p) + stack = {k: v for k, v in {"server": p.server, "cdn": p.cdn}.items() if v} + name = found.journey.name + evidence = list(diag.evidence) + starved = interference if interference and interference.get("journey") == name else None + if found.together and not starved: + alone = f" (it held {held_alone} users alone)" if held_alone is not None else "" + evidence.insert(0, f"{name} held up on its own{alone} but broke when the journeys " + f"ran together: {p.onset_users} users in total") + if found.shared: + evidence.append(f"steps {_names([str(i) for i in found.shared])} all call " + f"{p.culprit_route}, so their numbers under load are combined") + if found.samples < MIN_SAMPLES: + evidence.append(f"no 1-user baseline: only {found.samples} of {found.walks} replays " + "reached this step after the ramp") + if starved: + evidence.append(f"with {', '.join(starved['with'])} running too, {name} broke at " + f"{starved['users_at_break']} of its own users; alone it held " + f"{starved['held_alone']}: they compete for something they share") + elif interference: + evidence.append(f"also: with {', '.join(interference['with'])} running too, " + f"{interference['journey']} broke at {interference['users_at_break']} " + f"of its own users; alone it held {interference['held_alone']}") + return { + "culprit_route": p.culprit_route, + "culprit_journey": name, + "culprit_step": {"index": found.index, + "name": found.journey.steps[found.index - 1].name, + "route": p.culprit_route}, + "bottleneck_class": diag.bottleneck_class, + "confidence": diag.confidence, + "summary": diag.summary, + "evidence": evidence, + "remediation": remediate(diag, stack), + "stack": stack, + "interference": starved, + } + + +async def diagnose_run(worst: JourneyRun, runs: list[JourneyRun], base_url: str, *, + together: RunReport | None = None, combined: dict | None = None, + **kwargs) -> dict | None: + """The `investigation` block for a journey run: the step of the first + journey to break alone; if every journey held alone but they broke ramped + together, the step that broke in the combined run. None if nothing broke.""" + interference = (combined or {}).get("interference") + if worst.report is not None and worst.report.onset_users is not None: + found = await gather_step_probes(worst.journey, worst.report, base_url, **kwargs) + held_alone = None + elif together is not None and together.onset_users is not None and combined \ + and combined.get("culprit_journey"): + name = combined["culprit_journey"] + run = next((r for r in runs if r.journey.name == name and r.report is not None), None) + if run is None: + return None + found = await gather_step_probes(run.journey, together, base_url, + prefix=f"{name}{SEP}", **kwargs) + held_alone = run.report.max_tested + else: + return None + if found is None: + return None + return diagnose_step(found, interference=interference, held_alone=held_alone) diff --git a/cli/src/prescale_cli/render.py b/cli/src/prescale_cli/render.py index ef37f61..34ba840 100644 --- a/cli/src/prescale_cli/render.py +++ b/cli/src/prescale_cli/render.py @@ -258,15 +258,21 @@ def render_investigation(result: dict) -> None: if not inv: return color = {"high": "red", "medium": "yellow", "low": "blue"}.get(inv["confidence"], "yellow") + step = inv.get("culprit_step") + if step: + where = (f"{escape(inv['culprit_journey'])}, step {step['index']} " + f"[bold]{escape(step['name'])}[/bold] [dim]({escape(step['route'])})[/dim]") + else: + where = escape(inv["culprit_route"]) lines = [ - f"[bold]Likely cause:[/bold] {inv['summary']}", + f"[bold]Likely cause:[/bold] {escape(inv['summary'])}", f"Bottleneck [bold]{inv['bottleneck_class']}[/bold] " - f"([{color}]{inv['confidence']} confidence[/{color}]) · culprit {inv['culprit_route']}", + f"([{color}]{inv['confidence']} confidence[/{color}]) · culprit {where}", ] if inv.get("evidence"): lines.append("") lines.append("[bold]Evidence[/bold]") - lines += [f" • {e}" for e in inv["evidence"]] + lines += [f" • {escape(e)}" for e in inv["evidence"]] if inv.get("remediation"): lines.append("") lines.append("[bold]Try this[/bold]") diff --git a/cli/src/prescale_cli/schema/result.schema.json b/cli/src/prescale_cli/schema/result.schema.json index 622f890..21f7cef 100644 --- a/cli/src/prescale_cli/schema/result.schema.json +++ b/cli/src/prescale_cli/schema/result.schema.json @@ -141,6 +141,23 @@ "description": "Present when `prescale investigate` diagnosed a failure.", "properties": { "culprit_route": { "type": "string" }, + "culprit_journey": { + "type": ["string", "null"], + "description": "Journey runs: the journey whose step broke first." + }, + "culprit_step": { + "type": ["object", "null"], + "description": "Journey runs: the step that broke first.", + "properties": { + "index": { "type": "integer", "description": "1-based." }, + "name": { "type": "string" }, + "route": { "type": "string" } + } + }, + "interference": { + "type": ["object", "null"], + "description": "Journey runs: set when the combined run starved the culprit journey (same shape as combined.interference)." + }, "bottleneck_class": { "type": "string" }, "confidence": { "type": "string", "enum": ["high", "medium", "low"] }, "summary": { "type": "string" }, diff --git a/cli/tests/test_journeys_diagnose.py b/cli/tests/test_journeys_diagnose.py new file mode 100644 index 0000000..effa1bd --- /dev/null +++ b/cli/tests/test_journeys_diagnose.py @@ -0,0 +1,375 @@ +"""Diagnosing the journey step that breaks first (`prescale investigate` on journeys).""" + +import asyncio +import json +import os +import socket +import subprocess +import sys +import time +from pathlib import Path + +import httpx +import jsonschema +import pytest +import yaml +from click.testing import CliRunner + +from prescale_cli.investigate import Probes, classify +from prescale_cli.journeys.diagnose import ( + MIN_SAMPLES, + diagnose_run, + diagnose_step, + gather_step_probes, +) +from prescale_cli.journeys.load import JourneyRun, combined_block, ramp +from prescale_cli.journeys.model import parse +from prescale_cli.journeys.replay import SEP, check +from prescale_cli.loadtest import analyze +from prescale_cli.main import cli +from prescale_cli.render import render_investigation + +BASE = "http://shop.test" +ORDER = "POST /api/carts/{{cart_id}}/order" +SHOP = { + "version": 1, "name": "checkout", "type": "checkout", + "steps": [ + {"name": "log in", "request": {"method": "POST", "path": "/api/login"}, + "expect": {"status": 200}, "extract": {"token": "body.token"}}, + {"name": "add to cart", + "request": {"method": "POST", "path": "/api/cart", + "headers": {"Authorization": "Bearer {{token}}"}}, + "expect": {"status": 201}, "extract": {"cart_id": "body.id"}}, + {"name": "place order", + "request": {"method": "POST", "path": "/api/carts/{{cart_id}}/order", + "headers": {"Authorization": "Bearer {{token}}"}}, + "expect": {"status": 201}}, + ], +} +QUICKBUY = dict(SHOP, name="quickbuy") +FAST = {"seconds": 0, "cooldown": 0} + + +def _shop(pool: int, fail_status: int = 500): + """Orders fail with FAIL_STATUS once more than POOL are in flight; an order + needs the cart id and token from the earlier steps, as on a real shop.""" + state = {"in_flight": 0, "orders_without_state": 0} + + async def handler(request: httpx.Request) -> httpx.Response: + path = request.url.path + if path == "/api/login": + return httpx.Response(200, json={"token": "t"}) + if path == "/api/cart": + return httpx.Response(201, json={"id": 7}) + if path.endswith("/order"): + if path != "/api/carts/7/order" or request.headers.get("authorization") != "Bearer t": + state["orders_without_state"] += 1 + return httpx.Response(400) + if state["in_flight"] >= pool: + return httpx.Response(fail_status, text="pool exhausted") + state["in_flight"] += 1 + try: + await asyncio.sleep(0.02) + finally: + state["in_flight"] -= 1 + return httpx.Response(201, headers={"server": "gunicorn/21.2.0"}) + return httpx.Response(404) + + return httpx.MockTransport(handler), state + + +def _ramped(spec, transport, levels=(1, 2, 5, 10, 20)): + journey = parse(spec) + stages, _ = asyncio.run(ramp(journey, BASE, levels=list(levels), stage_seconds=0.2, + warmup=False, env={}, transport=transport)) + report = analyze(stages, latency_wall=2.0, error_threshold=0.02) + first = asyncio.run(check(journey, BASE, env={}, transport=transport)) + return JourneyRun(journey, first, report) + + +def _probes(run, transport, **kwargs): + return asyncio.run(gather_step_probes(run.journey, run.report, BASE, latency_wall=2.0, + error_threshold=0.02, env={}, transport=transport, + **{**FAST, **kwargs})) + + +def test_probes_the_step_that_broke_with_the_journeys_state(): + transport, state = _shop(pool=4) + run = _ramped(SHOP, transport) + found = _probes(run, transport) + p = found.probes + assert p.culprit_route == ORDER and found.index == 3 and not found.together + assert p.onset_users == run.report.onset_users + # the 1-user baseline walked log in and add to cart first, so the order carried them + assert found.samples >= MIN_SAMPLES and p.baseline_p95_ms >= 15 + assert state["orders_without_state"] == 0 + # the steps before it held while it failed + assert p.static_route == "log in, add to cart" and p.static_ok is True + assert p.server == "gunicorn/21.2.0" + + +def test_names_a_pool_when_one_step_fails_and_the_others_hold(): + transport, _ = _shop(pool=4) + run = _ramped(SHOP, transport) + inv = diagnose_step(_probes(run, transport)) + assert inv["culprit_journey"] == "checkout" + assert inv["culprit_step"] == {"index": 3, "name": "place order", "route": ORDER} + assert (inv["bottleneck_class"], inv["confidence"]) == ("connection_pool", "high") + assert "the journey's other steps held" in inv["summary"] + assert any("other steps (log in, add to cart) held" in e for e in inv["evidence"]) + assert any("pool" in r for r in inv["remediation"]) + assert inv["stack"] == {"server": "gunicorn/21.2.0"} + assert inv["interference"] is None + + +def test_a_4xx_under_load_is_not_read_as_a_pool_or_a_queue(): + transport, _ = _shop(pool=4, fail_status=409) + run = _ramped(SHOP, transport) + inv = diagnose_step(_probes(run, transport)) + assert inv["bottleneck_class"] == "unexpected_answer" + assert "unexpected 409" in inv["summary"] + # even when throughput flattened, which alone would read as a concurrency ceiling + probes = Probes(culprit_route=ORDER, onset_users=20, throughput_plateaued=True, + error_kinds={"unexpected 409": 9}, reference="the journey's other steps") + assert classify(probes).bottleneck_class == "unexpected_answer" + probes.error_kinds = {"missing extract": 4} + assert classify(probes).bottleneck_class == "unexpected_answer" + + +def test_steps_sharing_a_route_are_not_their_own_reference(): + spec = {"version": 1, "name": "cart", "type": "write", "steps": [ + {"name": "add one", "request": {"method": "POST", "path": "/api/cart"}, + "expect": {"status": 201}}, + {"name": "view", "request": {"path": "/api/view"}}, + {"name": "add two", "request": {"method": "POST", "path": "/api/cart"}, + "expect": {"status": 201}}, + ]} + state = {"in_flight": 0} + + async def handler(request): + if request.url.path == "/api/cart": + if state["in_flight"] >= 3: + return httpx.Response(500) + state["in_flight"] += 1 + try: + await asyncio.sleep(0.02) + finally: + state["in_flight"] -= 1 + return httpx.Response(201) + return httpx.Response(200) + + transport = httpx.MockTransport(handler) + run = _ramped(spec, transport) + found = _probes(run, transport) + assert found.probes.static_route == "view" and found.probes.static_ok is True + inv = diagnose_step(found) + assert inv["confidence"] == "high" + assert inv["culprit_step"]["index"] == 1 + assert any("steps 1 and 3 all call POST /api/cart" in e for e in inv["evidence"]) + + +def test_no_baseline_from_too_few_replays_and_it_says_so(): + transport, _ = _shop(pool=4) + run = _ramped(SHOP, transport) + dead = httpx.MockTransport(lambda request: httpx.Response(503)) + found = _probes(run, dead) + assert found.samples == 0 and found.probes.baseline_p95_ms is None + inv = diagnose_step(found) + assert any("no 1-user baseline: only 0 of 5 replays reached this step" in e + for e in inv["evidence"]) + + +def test_baseline_replays_stay_under_max_rps(): + transport, _ = _shop(pool=4) + run = _ramped(SHOP, transport) + start = time.monotonic() + found = _probes(run, transport, max_rps=30) + # 5 walks x 3 requests at 30 req/s is at least half a second + assert found.walks == 5 and time.monotonic() - start >= 0.45 + + +def _held_alone_but_not_together(): + """Two journeys sharing a 6-order pool: each holds 5 users alone; together, + at 10 users, they break.""" + transport, _ = _shop(pool=6) + runs = [_ramped(spec, transport, levels=(1, 2, 5)) for spec in (SHOP, QUICKBUY)] + assert all(r.report.onset_users is None for r in runs) + stages, _ = asyncio.run(ramp([r.journey for r in runs], BASE, levels=[1, 2, 5, 10, 20], + stage_seconds=0.2, warmup=False, env={}, transport=transport)) + together = analyze(stages, latency_wall=2.0, error_threshold=0.02) + assert together.onset_users is not None and SEP in together.culprit_route + return runs, together, combined_block(together, runs), transport + + +def test_journeys_that_only_break_together_are_diagnosed(): + runs, together, block, transport = _held_alone_but_not_together() + inv = asyncio.run(diagnose_run(runs[0], runs, BASE, together=together, combined=block, + latency_wall=2.0, error_threshold=0.02, env={}, + transport=transport, **FAST)) + assert inv is not None + assert inv["culprit_journey"] == block["culprit_journey"] + assert inv["culprit_step"]["name"] == "place order" + assert inv["interference"] == block["interference"] + name = block["culprit_journey"] + if block["interference"]: # said once, with its own numbers + assert any(f"{name} broke at" in e and "of its own users" in e for e in inv["evidence"]) + assert not any("held up on its own" in e for e in inv["evidence"]) + else: + assert inv["evidence"][0].startswith(f"{name} held up on its own (it held 5 users " + "alone) but broke when the journeys ran together") + # the onset level there counts every journey's users + assert any(e.endswith("users in total") for e in inv["evidence"]) + # fewer than 20 baseline samples: a median, and it says so + assert any(e.startswith("culprit median ") for e in inv["evidence"]) + + +def test_investigate_fail_under_gates_on_a_break_that_only_happens_together(capsys): + from prescale_cli.commands.investigate import _show + result = {"id": "x", "verdict": {"survives_users": 100, "onset_users": None}, + "combined": {"survives_users": 60, "onset_users": 75}} + with pytest.raises(SystemExit) as exit_: + _show(result, as_json=True, live_mode=False, upload=False, fail_under=80) + assert exit_.value.code == 1 + _show(result, as_json=True, live_mode=False, upload=False, fail_under=50) + + +def test_interference_on_the_culprit_journey_is_named_and_others_noted(): + transport, _ = _shop(pool=4) + run = _ramped(SHOP, transport) + found = _probes(run, transport) + starved = {"journey": "checkout", "users_at_break": 15, "held_alone": 20, + "with": ["browse"]} + inv = diagnose_step(found, interference=starved) + assert inv["interference"] == starved + assert any("with browse running too, checkout broke at 15 of its own users; " + "alone it held 20" in e for e in inv["evidence"]) + other = dict(starved, journey="search", **{"with": ["checkout"]}) + inv = diagnose_step(found, interference=other) + assert inv["interference"] is None + assert any(e.startswith("also: with checkout running too, search broke at 15") + for e in inv["evidence"]) + + +def test_a_journey_that_held_has_nothing_to_diagnose(): + transport, _ = _shop(pool=1000) + run = _ramped(SHOP, transport, levels=(1, 2, 5)) + assert run.report.onset_users is None + assert _probes(run, transport) is None + assert asyncio.run(diagnose_run(run, [run], BASE, latency_wall=2.0, error_threshold=0.02, + env={}, transport=transport, **FAST)) is None + + +def test_url_evidence_still_says_static(): + probes = Probes(culprit_route="GET /checkout", onset_users=50, static_route="GET /app.css", + static_ok=True, status_counts={500: 3}) + diag = classify(probes) + assert "static assets held" in diag.summary + assert "static GET /app.css held at 50 users" in diag.evidence + + +def test_the_panel_names_the_journey_and_step(capsys): + transport, _ = _shop(pool=4) + run = _ramped(SHOP, transport) + render_investigation({"investigation": diagnose_step(_probes(run, transport))}) + out = " ".join(capsys.readouterr().out.replace("│", " ").split()) + assert "checkout, step 3 place order" in out + assert "connection_pool" in out + + +# --- the command, end to end ------------------------------------------------- + +ROOT = Path(__file__).resolve().parents[2] + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture(scope="module") +def fragile_shop(): + """The example shop, for real: 25-connection pool behind place order.""" + port = _free_port() + proc = subprocess.Popen([sys.executable, str(ROOT / "examples" / "fragile-shop.py"), + str(port)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + for _ in range(100): + try: + socket.create_connection(("127.0.0.1", port), timeout=0.1).close() + break + except OSError: + time.sleep(0.05) + yield f"http://127.0.0.1:{port}" + finally: + proc.terminate() + proc.wait(timeout=5) + + +def test_investigate_json_end_to_end_on_the_example_shop(fragile_shop, tmp_path, monkeypatch): + monkeypatch.setenv("SHOP_PASSWORD", "demo") + res = CliRunner().invoke(cli, ["investigate", fragile_shop, "--json", "--journey", + "checkout", "--no-combined", "-u", "100", "-s", "0.5", + "--dir", str(ROOT / "examples" / "journeys"), + "--store", str(tmp_path), "--fail-under", "200"]) + assert res.exit_code == 1 # --fail-under: it breaks well below 200 + result = json.loads(res.stdout) # stdout is the Result alone + jsonschema.validate(result, json.loads( + (ROOT / "cli/src/prescale_cli/schema/result.schema.json").read_text())) + inv = result["investigation"] + assert inv["culprit_journey"] == "checkout" + assert inv["culprit_step"]["name"] == "place order" + assert inv["bottleneck_class"] == "connection_pool" + assert "demo" not in res.stdout + + +def test_run_and_ci_never_diagnose(fragile_shop, monkeypatch): + monkeypatch.setenv("SHOP_PASSWORD", "demo") + res = CliRunner().invoke(cli, ["run", fragile_shop, "--json", "--no-save", "--journey", + "checkout", "--no-combined", "-u", "50", "-s", "0.3", + "--dir", str(ROOT / "examples" / "journeys")]) + assert res.exit_code == 0, res.output + assert "investigation" not in json.loads(res.stdout) + + +def _dir(tmp_path, *specs): + d = tmp_path / "journeys" + d.mkdir(exist_ok=True) + for spec in specs: + (d / f"{spec['name']}.yaml").write_text(yaml.safe_dump(spec)) + return str(d) + + +def test_investigate_refuses_writes_on_a_non_local_host_before_any_request(tmp_path): + res = CliRunner().invoke(cli, ["investigate", "https://staging.example.com", + "--dir", _dir(tmp_path, SHOP), "--i-own-this"]) + assert res.exit_code == 2 + assert "checkout change data" in " ".join(res.output.split()) + assert "--allow-writes staging.example.com" in res.output + + +def test_investigate_refuses_unset_env_before_any_request(tmp_path): + spec = dict(SHOP, steps=[dict(SHOP["steps"][0], request={ + "method": "POST", "path": "/api/login", "json": {"password": "{{env.NOPE_UNSET}}"}})]) + os.environ.pop("NOPE_UNSET", None) + res = CliRunner().invoke(cli, ["investigate", "http://localhost:1", "--dir", + _dir(tmp_path, spec)]) + assert res.exit_code == 2 and "NOPE_UNSET" in res.output + + +def test_investigate_urls_and_path_skip_journeys(tmp_path, monkeypatch): + from prescale_cli.commands import investigate as inv_cmd + seen = {} + + async def fake(url, **kwargs): + seen["url"] = url + return {"id": "x", "verdict": {"survives_users": 1, "onset_users": None}, + "stages": [], "target": {"url": url, "routes": []}} + + monkeypatch.setattr(inv_cmd, "run_investigate", fake) + monkeypatch.setattr(inv_cmd, "run_journeys", lambda *a, **k: pytest.fail("journeys ran")) + for flags in (["--urls"], ["--path", "/x"]): + res = CliRunner().invoke(cli, ["investigate", "http://localhost:1", "--json", + "--dir", _dir(tmp_path, SHOP), *flags]) + assert res.exit_code == 0, res.output + assert seen["url"] == "http://localhost:1" diff --git a/docs/commands/investigate.mdx b/docs/commands/investigate.mdx index a68e45c..53f767b 100644 --- a/docs/commands/investigate.mdx +++ b/docs/commands/investigate.mdx @@ -70,6 +70,22 @@ Try this +## Journeys + +When your repo has journeys in `prescale/journeys/`, `investigate` ramps them the way `prescale run` does, then diagnoses **the journey step that breaks first**. The step is probed with the journey's own state: its 1-user latency comes from replaying the journey from the start (log in, add to cart, then place the order), so a step that needs a token or a cart id is measured the way a real user reaches it. The journey's other steps play the part static assets play for URLs: if they held while this step failed, the limit is something this step uses, such as a database pool. + +```text +🔬 Diagnosis +Likely cause: 5xx under load while the journey's other steps held — the app/backend is the wall (often a DB or upstream pool). +Bottleneck connection_pool (high confidence) · culprit checkout, step 3 place order (POST /api/carts/{{cart_id}}/order) +Evidence + • culprit p95 123ms at 1 user vs 178ms at 50 users + • the journey's other steps (log in, add to cart) held at 50 users + • errors under load: 5xx (25) +``` + +If the journeys ramped together starved the culprit journey (it broke with fewer of its own users than it held alone), the evidence says so too. If every journey held on its own but they broke ramped together, `investigate` diagnoses the step that broke in the combined run: that's journeys competing for something they share. A step that answers an unexpected 4xx under load (a 409, a 401) is classified as [`unexpected_answer`](/reference/bottleneck-types), not as a pool or a queue. The 1-user baseline replays the journey for about 2 seconds after a 1-second pause, within `--max-rps`; if too few replays reach the step, the evidence says there's no baseline. `--urls` investigates URLs instead. + ## Options @@ -124,6 +140,26 @@ Try this Exit with a non-zero status code if the app survives fewer than N concurrent users. Use this as a CI gate alongside `--i-own-this` and `--json`. + + Investigate only this journey (repeatable). Default: every journey in `prescale/journeys/`. + + + + Skip ramping the journeys together, so there's no interference check. + + + + Investigate URLs even when `prescale/journeys/` has journeys. + + + + Let journeys that change data (POST, PUT, PATCH, DELETE) run against this non-local host, e.g. a staging host. Repeatable. Without it, such journeys only run against local hosts. + + + + Where the journey files are. + + ## Safety `investigate` sends real traffic — a full ramp plus a handful of diagnostic probes — to your target. The same safety rules that apply to `prescale run` apply here: diff --git a/docs/reference/bottleneck-types.mdx b/docs/reference/bottleneck-types.mdx index 6dae5e5..3d0912f 100644 --- a/docs/reference/bottleneck-types.mdx +++ b/docs/reference/bottleneck-types.mdx @@ -141,6 +141,21 @@ The classifier is fully local and deterministic — no LLM, no network calls. It * Inspect the per-route breakdown in `verdict.stages[].routes` to see which route degrades fastest. * Check application logs and database/connection metrics around the `onset_users` level — the signal you need may not be visible from the outside. + + + **What it means** + + Journeys only. The step works for one user, but under load it answers something the journey doesn't expect: a 4xx such as 409, 401 or 403, or a response missing a value a later step needs. Usually users are conflicting over shared data (a row several users change at once, a lock, a unique constraint), or sessions and tokens stop holding up under load. + + **How it's detected** + + The step's most common failure under load is an unexpected 4xx or a missing extract. Confidence is `medium`: PreScale sees the answer, not the reason behind it. + + **How to fix it** + + * Check the app's logs for this step under load. A 409 or a unique-constraint error points at rows several users change at once: lock them, or retry. A 401 or 403 points at a session store or token cache that doesn't keep up. + * Make sure each virtual user's data is its own. A journey where every user changes the same account or item can conflict by design. + *** diff --git a/docs/reference/result-schema.mdx b/docs/reference/result-schema.mdx index 11a4886..fb58aec 100644 --- a/docs/reference/result-schema.mdx +++ b/docs/reference/result-schema.mdx @@ -300,6 +300,18 @@ The `investigation` object is added to the result when you run `prescale investi Ordered list of concrete fix recommendations, enriched with hints based on detected stack (e.g. gunicorn, nginx, pgbouncer). + + Journey runs only: the journey whose step broke first. + + + + Journey runs only: the step that broke first, as `index` (1-based), `name` and `route`. + + + + Journey runs only: set when the journeys ramped together starved the culprit journey. Same shape as `combined.interference`. + + Key-value map of detected infrastructure signals — typically `server` (e.g. `"gunicorn/21.2.0"`) and `cdn` (e.g. `"Cloudflare"`). Empty if nothing was detected.