From 6876c773ed0e8f50f5d947cbea37e362a9d19fd3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 07:35:14 +0000 Subject: [PATCH 1/4] feat(replica): --db neon gives the app a throwaway copy of the database --- CHANGELOG.md | 12 ++ README.md | 6 + cli/src/prescale_cli/commands/ci.py | 8 +- cli/src/prescale_cli/commands/investigate.py | 8 +- cli/src/prescale_cli/commands/replicaopts.py | 104 +++++++++- cli/src/prescale_cli/commands/run.py | 8 +- cli/src/prescale_cli/dbbranch.py | 155 ++++++++++++++ cli/src/prescale_cli/replica.py | 9 +- cli/src/prescale_cli/result.py | 8 + .../prescale_cli/schema/result.schema.json | 9 + cli/tests/test_dbbranch.py | 191 ++++++++++++++++++ docs/guides/replica-mode.mdx | 29 +++ 12 files changed, 531 insertions(+), 16 deletions(-) create mode 100644 cli/src/prescale_cli/dbbranch.py create mode 100644 cli/tests/test_dbbranch.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a95241..8d87f63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## Unreleased + +### Added +- **A throwaway copy of your database for replica mode (Neon).** `--db neon --db-parent + staging` on `run`, `investigate` and `ci` copies that Neon branch, starts the app (`--app`) + with `DATABASE_URL` pointing at the copy, and deletes the copy when the test ends, even on + errors or Ctrl-C. Journeys that write change the copy, never the parent, and the load runs on + the copy's own compute. Needs `NEON_API_KEY` and `NEON_PROJECT_ID`. The parent must be named, + and the connection string is never printed, saved or uploaded; saved runs record + `config.database` (provider, parent, database name). `--db-name`, `--db-env` and + `--db-pooled` cover multi-database branches, other variable names and Neon's pooler. + ## 1.2.0 — 2026-11-08 ### Added diff --git a/README.md b/README.md index b47038a..bf37fcf 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,12 @@ prescale investigate http://localhost:3000 --app "npm run build && npm start" PreScale runs the command in your project, waits until the URL answers (`--app-timeout`, default 600s, since builds take a while), runs the test, then stops the app and everything it started, even on errors or Ctrl-C. The app uses your environment, so point `DATABASE_URL` and API keys at a dev or staging database. Results say they were measured on a local replica (no serverless cold starts), and the command itself is never saved or uploaded. +**A throwaway copy of your database** (Neon). Add `--db neon --db-parent staging` and PreScale copies that Neon branch (copy-on-write: seconds, whatever its size), starts your app with `DATABASE_URL` pointing at the copy, and deletes the copy when the test ends, even on errors or Ctrl-C. Journeys that write change the copy, never staging, and the load runs on the copy's own compute. It needs `NEON_API_KEY` and `NEON_PROJECT_ID`. You always name the branch to copy (the default branch is often production), and the copy's connection string is never printed, saved or uploaded. + +```bash +prescale run http://localhost:3000 --app "npm run build && npm start" --db neon --db-parent staging +``` + In CI it needs no deploy at all: build the pull request in the job and test it there. ```yaml diff --git a/cli/src/prescale_cli/commands/ci.py b/cli/src/prescale_cli/commands/ci.py index 065a730..eb9d8c6 100644 --- a/cli/src/prescale_cli/commands/ci.py +++ b/cli/src/prescale_cli/commands/ci.py @@ -29,7 +29,12 @@ ) from prescale_cli.cloud import CloudError, parse_headers, require_credentials, upload_result from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes -from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica +from prescale_cli.commands.replicaopts import ( + app_option, + app_timeout_option, + db_options, + start_replica, +) from prescale_cli.commands.run import run_journeys from prescale_cli.hosts import PERMISSION_ENV from prescale_cli.investigate import investigate as run_investigate @@ -85,6 +90,7 @@ def _annotate(level: str, message: str) -> None: @dir_option @app_option @app_timeout_option +@db_options @click.option("--store", "store", type=click.Path(file_okay=False), default=None, help="Directory for saved runs (default: ./.prescale).") def ci(url: str | None, target: str, paths: tuple[str, ...], header_values: tuple[str, ...], diff --git a/cli/src/prescale_cli/commands/investigate.py b/cli/src/prescale_cli/commands/investigate.py index 9ef8eff..aab0482 100644 --- a/cli/src/prescale_cli/commands/investigate.py +++ b/cli/src/prescale_cli/commands/investigate.py @@ -15,7 +15,12 @@ from prescale_cli.commands.audit import _ICONS as _AUDIT_ICONS from prescale_cli.commands.hostcheck import confirm_host, host_permission_option from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes -from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica +from prescale_cli.commands.replicaopts import ( + app_option, + app_timeout_option, + db_options, + start_replica, +) from prescale_cli.commands.run import _journeys_to_run, run_journeys from prescale_cli.commands.upload import header_option, upload_after_run, upload_option from prescale_cli.investigate import _ERROR_THRESHOLD, _LATENCY_WALL_S, audit_block @@ -65,6 +70,7 @@ @dir_option @app_option @app_timeout_option +@db_options @host_permission_option @header_option @upload_option diff --git a/cli/src/prescale_cli/commands/replicaopts.py b/cli/src/prescale_cli/commands/replicaopts.py index f917545..10e8088 100644 --- a/cli/src/prescale_cli/commands/replicaopts.py +++ b/cli/src/prescale_cli/commands/replicaopts.py @@ -1,17 +1,25 @@ -"""`--app` / `--app-timeout`: replica mode for `run`, `investigate` and `ci`. +"""`--app` / `--app-timeout` (replica mode) and `--db` (a throwaway database +copy for it) for `run`, `investigate` and `ci`. -The app is started through Click's context, so it's stopped when the command -ends however it ends: a verdict, an error, or Ctrl-C. +The app and its database copy are started through Click's context, so they're +stopped and deleted when the command ends however it ends: a verdict, an error, +or Ctrl-C. The copy is made first and deleted last, after the app has stopped. """ from __future__ import annotations +import functools + import click from rich.console import Console from rich.panel import Panel +from prescale_cli.dbbranch import DBBranchError, neon_branch, neon_settings from prescale_cli.replica import DEFAULT_TIMEOUT, ReplicaError, replica, require_local +# Where the --db choices and the copy's (non-secret) description live for the command. +DB_META = "prescale.db" + app_option = click.option( "--app", "app", default=None, metavar="COMMAND", help='Start your app with COMMAND (e.g. "npm run build && npm start"), test it on the ' @@ -22,27 +30,105 @@ help="Seconds to wait for the app to answer (builds can take minutes).") +def db_options(fn): + """--db, --db-parent, --db-name, --db-env, --db-pooled. They need --app: only + an app PreScale starts can be pointed at the copy.""" + @click.option("--db", "db", type=click.Choice(["neon"]), default=None, + help="With --app: give the app its own throwaway copy of the database (a " + "Neon branch), deleted when the test ends. Needs NEON_API_KEY and " + "NEON_PROJECT_ID.") + @click.option("--db-parent", "db_parent", default=None, metavar="BRANCH", + help="The Neon branch to copy, e.g. staging. Required with --db: the " + "default branch is often production, so PreScale never picks it.") + @click.option("--db-name", "db_name", default=None, metavar="NAME", + help="The database in the branch, if it has more than one.") + @click.option("--db-env", "db_env", default="DATABASE_URL", show_default=True, + metavar="VAR", help="The environment variable the app reads its database " + "URL from.") + @click.option("--db-pooled", "db_pooled", is_flag=True, + help="Give the app Neon's pooled connection URL (through its pooler).") + @functools.wraps(fn) + def wrapper(*args, db, db_parent, db_name, db_env, db_pooled, **kwargs): + if db: + if not kwargs.get("app"): + raise click.UsageError("--db needs --app: PreScale can only point an app it " + "starts at the database copy.") + if not db_parent: + raise click.UsageError("--db neon needs --db-parent BRANCH, the branch to " + "copy (e.g. staging).") + click.get_current_context().meta[DB_META] = { + "provider": db, "parent": db_parent, "name": db_name, "env": db_env, + "pooled": db_pooled} + return fn(*args, **kwargs) + return wrapper + + +def _start_database(console: Console, quiet: bool) -> dict[str, str]: + """Make the database copy if --db asked for one; the app's extra environment.""" + ctx = click.get_current_context() + want = ctx.meta.get(DB_META) + if not want: + return {} + try: + api_key, project = neon_settings() + except DBBranchError as exc: + console.print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) from None + + def left_behind(name, exc): + Console(stderr=True).print(f"[yellow]⚠ Couldn't delete the Neon branch {name}: {exc} " + "Delete it in the Neon console.[/yellow]") + copy = neon_branch(want["parent"], api_key=api_key, project=project, + database=want["name"], pooled=want["pooled"], + on_delete_failed=left_behind) + try: + if quiet: + branch = ctx.with_resource(copy) + else: + with console.status(f"Copying the Neon branch {want['parent']}…"): + branch = ctx.with_resource(copy) + except DBBranchError as exc: + console.print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) from None + # What the Result may say about it: never the URL. + want["copy"] = {"provider": "neon", "parent": branch.parent, "database": branch.database} + if not quiet: + console.print(f"[green]✓[/green] a throwaway copy of {branch.parent} is ready " + f"([dim]{branch.name}[/dim]); it's deleted when the test ends") + return {want["env"]: branch.uri} + + +def database_info() -> dict | None: + """The database copy this command's app ran against, for the Result.""" + ctx = click.get_current_context(silent=True) + want = ctx.meta.get(DB_META) if ctx else None + return (want or {}).get("copy") + + def start_replica(app: str, url: str, *, timeout: float, console: Console, quiet: bool) -> None: - """Start the app for this command, or exit with the reason it couldn't start.""" + """Start the app for this command (with its database copy, if --db asked for + one), or exit with the reason it couldn't start.""" try: require_local(url) except ReplicaError as exc: console.print(f"[red]Error:[/red] {exc}") raise SystemExit(1) from None + env = _start_database(console, quiet) if not quiet: console.print(Panel( - f"Starting your app with [bold]{app}[/bold] and waiting for {url}.\n" - "It runs with your environment: point DATABASE_URL and API keys at dev or " - "staging, not production.\n" + f"Starting your app with [bold]{app}[/bold] and waiting for {url}.\n" + + ("Its database is the throwaway copy. " if env else + "It runs with your environment: point DATABASE_URL and API keys at dev or " + "staging, not production.\n") + "PreScale stops it when the test ends. The command is never saved or uploaded.", title="Replica mode", border_style="cyan")) ctx = click.get_current_context() try: if quiet: - ctx.with_resource(replica(app, url, timeout=timeout)) + ctx.with_resource(replica(app, url, timeout=timeout, env=env)) else: with console.status("Building and starting your app…"): - ctx.with_resource(replica(app, url, timeout=timeout)) + ctx.with_resource(replica(app, url, timeout=timeout, env=env)) except ReplicaError as exc: console.print(f"[red]Error:[/red] {exc}") raise SystemExit(1) from None diff --git a/cli/src/prescale_cli/commands/run.py b/cli/src/prescale_cli/commands/run.py index 5255ac3..9b50ac4 100644 --- a/cli/src/prescale_cli/commands/run.py +++ b/cli/src/prescale_cli/commands/run.py @@ -19,7 +19,12 @@ from prescale_cli.cloud import CloudError, require_credentials from prescale_cli.commands.hostcheck import confirm_host, host_permission_option from prescale_cli.commands.journeys import dir_option, refuse_missing_env, refuse_unallowed_writes -from prescale_cli.commands.replicaopts import app_option, app_timeout_option, start_replica +from prescale_cli.commands.replicaopts import ( + app_option, + app_timeout_option, + db_options, + start_replica, +) from prescale_cli.commands.upload import header_option, upload_after_run, upload_option from prescale_cli.fixprompt import fix_prompt from prescale_cli.journeys.diagnose import diagnose_run @@ -112,6 +117,7 @@ @dir_option @app_option @app_timeout_option +@db_options @host_permission_option @header_option @upload_option diff --git a/cli/src/prescale_cli/dbbranch.py b/cli/src/prescale_cli/dbbranch.py new file mode 100644 index 0000000..5c23d51 --- /dev/null +++ b/cli/src/prescale_cli/dbbranch.py @@ -0,0 +1,155 @@ +"""Database copies for replica mode: a throwaway Neon branch for each run. + +`--db neon --db-parent staging` makes a copy of the `staging` branch (Neon +branches are copy-on-write, so this takes seconds whatever the size), gives it +its own compute, starts the app with DATABASE_URL pointing at it, and deletes +the copy when the test ends, however it ends. Journeys that write change the +copy, never the database the copy came from, and the load runs on the copy's +own compute, so it never competes with the parent's traffic. + +The parent must be named: the project's default branch is often production, so +PreScale never picks it silently. The connection string is a secret: it goes +into the app's environment and nowhere else (not the terminal, the Result, an +upload or a log). +""" + +from __future__ import annotations + +import os +import secrets +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from datetime import datetime, timezone + +import httpx + +NEON_API = "https://console.neon.tech/api/v2" +READY_TIMEOUT = 120.0 +_POLL_S = 1.0 +_NAME_PREFIX = "prescale-" + + +class DBBranchError(Exception): + """The copy couldn't be made, or a Neon call failed.""" + + +@dataclass +class Branch: + id: str + name: str + parent: str + database: str + uri: str = field(repr=False) # a secret: never printed, saved or uploaded + + +def neon_settings(env: dict[str, str] | None = None) -> tuple[str, str]: + """NEON_API_KEY and NEON_PROJECT_ID, or a clear error naming what's missing.""" + env = os.environ if env is None else env + missing = [n for n in ("NEON_API_KEY", "NEON_PROJECT_ID") if not env.get(n)] + if missing: + raise DBBranchError(f"--db neon needs {' and '.join(missing)} in the environment " + "(a Neon API key, and the ID of the project to copy from).") + return env["NEON_API_KEY"], env["NEON_PROJECT_ID"] + + +def _call(client: httpx.Client, method: str, path: str, **kwargs) -> dict: + try: + resp = client.request(method, path, **kwargs) + except httpx.HTTPError as exc: + raise DBBranchError(f"Couldn't reach Neon ({exc.__class__.__name__}).") from exc + if resp.status_code == 401: + raise DBBranchError("Neon refused NEON_API_KEY (401). Check the key is current.") + if resp.status_code == 404: + raise DBBranchError("Neon has no such project or branch (404). Check NEON_PROJECT_ID.") + if resp.status_code >= 400: + detail = "" + try: + detail = str(resp.json().get("message") or "")[:200] + except ValueError: + pass + raise DBBranchError(f"Neon answered {resp.status_code} to {method} {path.split('?')[0]}" + + (f": {detail}" if detail else ".")) + return resp.json() if resp.content else {} + + +def _find_parent(client: httpx.Client, project: str, parent: str) -> dict: + branches = _call(client, "GET", f"/projects/{project}/branches").get("branches") or [] + match = next((b for b in branches if parent in (b.get("name"), b.get("id"))), None) + if match is None: + names = ", ".join(sorted(b.get("name", "?") for b in branches)) or "none" + raise DBBranchError(f"No branch named {parent!r} in the Neon project. Branches: {names}.") + return match + + +def _wait(client: httpx.Client, project: str, operations: list[dict], timeout: float) -> None: + pending = {op["id"] for op in operations if op.get("status") != "finished"} + deadline = time.monotonic() + timeout + while pending: + if time.monotonic() > deadline: + raise DBBranchError(f"The Neon branch wasn't ready within {timeout:g}s.") + for op_id in list(pending): + op = _call(client, "GET", f"/projects/{project}/operations/{op_id}").get( + "operation") or {} + status = op.get("status") + if status == "finished": + pending.discard(op_id) + elif status in ("failed", "error", "cancelled"): + raise DBBranchError(f"Neon couldn't create the branch (operation {status}).") + if pending: + time.sleep(_POLL_S) + + +def _database(client: httpx.Client, project: str, branch_id: str, + name: str | None) -> tuple[str, str]: + dbs = _call(client, "GET", f"/projects/{project}/branches/{branch_id}/databases").get( + "databases") or [] + if name: + db = next((d for d in dbs if d.get("name") == name), None) + if db is None: + raise DBBranchError(f"The branch has no database named {name!r}. Databases: " + f"{', '.join(d.get('name', '?') for d in dbs) or 'none'}.") + elif len(dbs) == 1: + db = dbs[0] + else: + raise DBBranchError("The branch has several databases " + f"({', '.join(d.get('name', '?') for d in dbs)}): " + "pick one with --db-name.") + return db["name"], db["owner_name"] + + +@contextmanager +def neon_branch(parent: str, *, api_key: str, project: str, database: str | None = None, + pooled: bool = False, timeout: float = READY_TIMEOUT, + transport: httpx.BaseTransport | None = None, on_delete_failed=None): + """Make a throwaway branch of PARENT, yield it, then delete it (also on + errors and Ctrl-C). ON_DELETE_FAILED(branch, error) is told if Neon won't + delete it, so the user can remove it by name.""" + client = httpx.Client(base_url=NEON_API, timeout=30.0, transport=transport, + headers={"Authorization": f"Bearer {api_key}", + "Accept": "application/json"}) + created = None + try: + source = _find_parent(client, project, parent) + stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + name = f"{_NAME_PREFIX}{stamp}-{secrets.token_hex(3)}" + made = _call(client, "POST", f"/projects/{project}/branches", json={ + "branch": {"parent_id": source["id"], "name": name}, + "endpoints": [{"type": "read_write"}], + }) + created = made["branch"] + _wait(client, project, made.get("operations") or [], timeout) + db_name, role = _database(client, project, created["id"], database) + uri = _call(client, "GET", f"/projects/{project}/connection_uri", params={ + "branch_id": created["id"], "database_name": db_name, "role_name": role, + "pooled": "true" if pooled else "false"})["uri"] + yield Branch(id=created["id"], name=name, parent=source.get("name") or parent, + database=db_name, uri=uri) + finally: + if created is not None: + try: + _call(client, "DELETE", f"/projects/{project}/branches/{created['id']}") + except DBBranchError as exc: + if on_delete_failed: + on_delete_failed(created.get("name") or created["id"], exc) + client.close() diff --git a/cli/src/prescale_cli/replica.py b/cli/src/prescale_cli/replica.py index 4e2e2a8..e13554c 100644 --- a/cli/src/prescale_cli/replica.py +++ b/cli/src/prescale_cli/replica.py @@ -52,7 +52,7 @@ def require_local(url: str) -> None: @contextmanager def replica(command: str, url: str, *, timeout: float = DEFAULT_TIMEOUT, cwd: str | None = None, - on_wait=None): + on_wait=None, env: dict[str, str] | None = None): """Start `command`, wait for `url` to answer, yield, then stop the app and everything it started (also on errors and Ctrl-C).""" require_local(url) @@ -60,7 +60,7 @@ def replica(command: str, url: str, *, timeout: float = DEFAULT_TIMEOUT, cwd: st raise ReplicaError(f"Something is already running at {url}. Stop it, or point the " "app at another port, so PreScale tests the build it starts.") log = tempfile.NamedTemporaryFile(prefix="prescale-app-", suffix=".log", delete=False) - process = _start(command, cwd, log) + process = _start(command, cwd, log, env) try: _wait_until_ready(process, url, timeout, log.name, on_wait) yield Replica(pid=process.pid, log_path=log.name) @@ -69,14 +69,15 @@ def replica(command: str, url: str, *, timeout: float = DEFAULT_TIMEOUT, cwd: st log.close() -def _start(command: str, cwd: str | None, log) -> subprocess.Popen: +def _start(command: str, cwd: str | None, log, env: dict[str, str] | None = None + ) -> subprocess.Popen: # A new process group, so stopping it also stops what npm or a shell started. group = ({"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} if sys.platform == "win32" else {"start_new_session": True}) try: return subprocess.Popen(command, shell=True, cwd=cwd, stdout=log, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, - env=os.environ.copy(), **group) + env={**os.environ, **(env or {})}, **group) except OSError as exc: raise ReplicaError(f"Couldn't start the app: {exc}") from exc diff --git a/cli/src/prescale_cli/result.py b/cli/src/prescale_cli/result.py index 2742443..5f1d772 100644 --- a/cli/src/prescale_cli/result.py +++ b/cli/src/prescale_cli/result.py @@ -75,6 +75,14 @@ def build_result(report: RunReport, *, url: str, targets: list[str], if replica: config = {**config, "replica": True} warning = f"{warning} {REPLICA_NOTE}" if warning else REPLICA_NOTE + from prescale_cli.commands.replicaopts import database_info + database = database_info() + if database: + config = {**config, "database": database} + note = (f"Its database was a throwaway Neon copy of {database['parent']} on its " + "own compute, which may be sized differently from production's.") + warning = f"{warning} {note}" + return { "schema_version": SCHEMA_VERSION, "kind": "run", diff --git a/cli/src/prescale_cli/schema/result.schema.json b/cli/src/prescale_cli/schema/result.schema.json index 1ad8816..f9fcf03 100644 --- a/cli/src/prescale_cli/schema/result.schema.json +++ b/cli/src/prescale_cli/schema/result.schema.json @@ -46,6 +46,15 @@ "type": "boolean", "description": "True when PreScale started the app's own build locally (--app) and tested that, not a deployed host." }, + "database": { + "type": "object", + "description": "With --db: the throwaway database copy the app ran against. Never includes a connection string.", + "properties": { + "provider": { "type": "string", "enum": ["neon"] }, + "parent": { "type": "string", "description": "The branch that was copied." }, + "database": { "type": "string" } + } + }, "mode": { "type": "string", "enum": ["journeys"], diff --git a/cli/tests/test_dbbranch.py b/cli/tests/test_dbbranch.py new file mode 100644 index 0000000..503f50c --- /dev/null +++ b/cli/tests/test_dbbranch.py @@ -0,0 +1,191 @@ +"""`--db neon`: a throwaway Neon branch for the replica, made and deleted around the run.""" + +import functools +import json +import sys + +import httpx +import pytest +from click.testing import CliRunner + +from prescale_cli import dbbranch +from prescale_cli.dbbranch import DBBranchError, neon_branch, neon_settings +from prescale_cli.main import cli + +URI = "postgresql://owner:s3cr3t-pw@ep-copy-123.us-east-2.aws.neon.tech/shop?sslmode=require" + + +class FakeNeon: + """Enough of Neon's API: branches, operations, databases, connection URI.""" + + def __init__(self, *, databases=("shop",), op_status=("running", "finished"), + branches=("main", "staging")): + self.calls: list[tuple[str, str]] = [] + self.created: dict | None = None + self.deleted: list[str] = [] + self.databases = databases + self.op_status = list(op_status) + self.branches = [{"id": f"br-{n}", "name": n} for n in branches] + + def handler(self, request: httpx.Request) -> httpx.Response: + path, method = request.url.path.removeprefix("/api/v2"), request.method + self.calls.append((method, path)) + assert request.headers["authorization"] == "Bearer key-123" + if method == "GET" and path == "/projects/p1/branches": + return httpx.Response(200, json={"branches": self.branches}) + if method == "POST" and path == "/projects/p1/branches": + body = json.loads(request.content) + self.created = body + return httpx.Response(201, json={ + "branch": {"id": "br-copy", "name": body["branch"]["name"]}, + "operations": [{"id": "op1", "status": "running"}]}) + if method == "GET" and path == "/projects/p1/operations/op1": + status = self.op_status.pop(0) if self.op_status else "finished" + return httpx.Response(200, json={"operation": {"id": "op1", "status": status}}) + if method == "GET" and path == "/projects/p1/branches/br-copy/databases": + return httpx.Response(200, json={"databases": [ + {"name": d, "owner_name": "owner"} for d in self.databases]}) + if method == "GET" and path == "/projects/p1/connection_uri": + assert request.url.params["branch_id"] == "br-copy" + assert request.url.params["role_name"] == "owner" + self.uri_params = dict(request.url.params) + return httpx.Response(200, json={"uri": URI}) + if method == "DELETE" and path == "/projects/p1/branches/br-copy": + self.deleted.append("br-copy") + return httpx.Response(200, json={}) + return httpx.Response(404, json={"message": "nope"}) + + def transport(self): + return httpx.MockTransport(self.handler) + + +@pytest.fixture(autouse=True) +def fast(monkeypatch): + monkeypatch.setattr(dbbranch, "_POLL_S", 0) + + +def _branch(fake, parent="staging", **kw): + return neon_branch(parent, api_key="key-123", project="p1", transport=fake.transport(), + **kw) + + +def test_copies_the_named_branch_waits_until_ready_and_deletes_it(): + fake = FakeNeon() + with _branch(fake) as branch: + assert branch.uri == URI and branch.parent == "staging" and branch.database == "shop" + assert fake.deleted == [] + body = fake.created + assert body["branch"]["parent_id"] == "br-staging" + assert body["branch"]["name"].startswith("prescale-") + assert body["endpoints"] == [{"type": "read_write"}] + assert fake.calls.count(("GET", "/projects/p1/operations/op1")) == 2 # polled until finished + assert fake.uri_params["pooled"] == "false" + assert fake.deleted == ["br-copy"] + assert URI not in repr(branch) and "s3cr3t" not in repr(branch) + + +def test_the_copy_is_deleted_when_the_test_fails(): + fake = FakeNeon() + with pytest.raises(KeyboardInterrupt), _branch(fake): + raise KeyboardInterrupt + assert fake.deleted == ["br-copy"] + + +def test_a_failed_creation_still_deletes_what_was_made(): + fake = FakeNeon(op_status=("failed",)) + with pytest.raises(DBBranchError, match="couldn't create the branch"), _branch(fake): + pass + assert fake.deleted == ["br-copy"] + + +def test_a_missing_parent_says_which_branches_exist_and_makes_nothing(): + fake = FakeNeon() + with pytest.raises(DBBranchError, match="No branch named 'qa'.*main, staging"), \ + _branch(fake, parent="qa"): + pass + assert fake.created is None and fake.deleted == [] + + +def test_several_databases_need_a_name_and_pooled_is_passed(): + fake = FakeNeon(databases=("shop", "analytics")) + with pytest.raises(DBBranchError, match="--db-name"), _branch(fake): + pass + fake = FakeNeon(databases=("shop", "analytics")) + with _branch(fake, database="analytics", pooled=True) as branch: + assert branch.database == "analytics" + assert fake.uri_params["pooled"] == "true" + + +def test_a_refused_key_is_named(): + def refuse(request): + return httpx.Response(401, json={"message": "invalid"}) + with pytest.raises(DBBranchError, match="refused NEON_API_KEY"), \ + neon_branch("staging", api_key="k", project="p1", + transport=httpx.MockTransport(refuse)): + pass + + +def test_settings_name_what_is_missing(): + with pytest.raises(DBBranchError, match="NEON_API_KEY and NEON_PROJECT_ID"): + neon_settings({}) + assert neon_settings({"NEON_API_KEY": "k", "NEON_PROJECT_ID": "p"}) == ("k", "p") + + +# --- the command --------------------------------------------------------------- + +def test_db_needs_app_and_a_parent(): + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--db", "neon"]) + assert res.exit_code == 2 and "--db needs --app" in res.output + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--db", "neon", "--app", "x"]) + assert res.exit_code == 2 and "--db-parent" in res.output + + +def test_missing_neon_settings_stop_before_the_app_starts(monkeypatch, tmp_path): + monkeypatch.delenv("NEON_API_KEY", raising=False) + marker = tmp_path / "started" + res = CliRunner().invoke(cli, ["run", "http://localhost:1", "--app", f"touch {marker}", + "--db", "neon", "--db-parent", "staging", "--urls"]) + assert res.exit_code == 1 and "NEON_API_KEY" in res.output + assert not marker.exists() + + +APP = """ +import http.server, os, sys +open(sys.argv[2], "w").write(os.environ.get("DATABASE_URL", "")) +class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200); self.send_header("Content-Length", "2"); self.end_headers() + self.wfile.write(b"ok") + def log_message(self, *a): pass +http.server.ThreadingHTTPServer(("127.0.0.1", int(sys.argv[1])), H).serve_forever() +""" + + +def test_run_gives_the_app_the_copy_and_never_shows_its_url(monkeypatch, tmp_path): + import socket + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + fake = FakeNeon() + from prescale_cli.commands import replicaopts + monkeypatch.setattr(replicaopts, "neon_branch", + functools.partial(neon_branch, transport=fake.transport())) + monkeypatch.setenv("NEON_API_KEY", "key-123") + monkeypatch.setenv("NEON_PROJECT_ID", "p1") + (tmp_path / "app.py").write_text(APP) + seen = tmp_path / "seen.txt" + store = tmp_path / "store" + res = CliRunner().invoke(cli, [ + "run", f"http://127.0.0.1:{port}", "--urls", "-u", "2", "-s", "0.2", + "--app", f"{sys.executable} {tmp_path / 'app.py'} {port} {seen}", + "--db", "neon", "--db-parent", "staging", "--json", "--store", str(store)]) + assert res.exit_code == 0, res.output + assert seen.read_text() == URI # the app got the copy + assert fake.deleted == ["br-copy"] # and the copy is gone + result = json.loads(res.stdout) + assert result["config"]["database"] == {"provider": "neon", "parent": "staging", + "database": "shop"} + assert "throwaway Neon copy of staging" in result["warning"] + saved = "".join(p.read_text() for p in store.rglob("*.json")) + for text in (res.output, saved): + assert "s3cr3t" not in text and "ep-copy-123" not in text diff --git a/docs/guides/replica-mode.mdx b/docs/guides/replica-mode.mdx index 5022678..c00e1f7 100644 --- a/docs/guides/replica-mode.mdx +++ b/docs/guides/replica-mode.mdx @@ -41,6 +41,35 @@ The URL must be local. If something is already answering there, PreScale stops i testing it, so you always test the build it started. If the app crashes or never comes up, you get its exit code and the end of its log. +## A throwaway copy of your database + +With `--db neon`, the app doesn't use your staging database at all. PreScale copies a Neon +branch you name, points the app at the copy, and deletes the copy when the test ends. + +```bash +export NEON_API_KEY=... # a Neon API key +export NEON_PROJECT_ID=... # the project to copy from +prescale run http://localhost:3000 --app "npm run build && npm start" \ + --db neon --db-parent staging +``` + +- **Safe writes.** Journeys that write (sign-ups, carts, orders) change the copy, never + `staging`. +- **Real data, real query plans.** Neon branches are copy-on-write, so the copy has all of + the parent's data within seconds, whatever its size. +- **Its own compute.** The load runs on the copy's compute, not the parent's. That compute may + be sized differently from production's, and results say so. +- **Always cleaned up.** The copy is deleted however the run ends. If Neon refuses, PreScale + names the branch (`prescale-...`) so you can delete it. +- **You choose the parent.** `--db-parent` is required: the default branch is often + production, so PreScale never picks it for you. +- **The connection string stays secret.** It goes into the app's environment only (as + `DATABASE_URL`, or the variable you name with `--db-env`), never into the terminal, the + saved run or an upload. + +Options: `--db-name` picks the database when the branch has more than one, and `--db-pooled` +gives the app Neon's pooled URL. + ## In CI, on every pull request In CI, replica mode needs no deploy at all: the [GitHub Action](/guides/github-action) From b61479002cc67263888e46c0525a1a7e66c62a0d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 18:26:22 +0000 Subject: [PATCH 2/4] fix(replica): mask the copy's secrets in app logs, point every DB variable at it, never orphan it --- CHANGELOG.md | 8 +- cli/src/prescale_cli/commands/db.py | 53 +++++++ cli/src/prescale_cli/commands/replicaopts.py | 60 ++++++-- cli/src/prescale_cli/dbbranch.py | 143 ++++++++++++++++--- cli/src/prescale_cli/main.py | 2 + cli/src/prescale_cli/replica.py | 35 +++-- cli/tests/test_dbbranch.py | 131 +++++++++++++++-- docs/guides/replica-mode.mdx | 17 +++ 8 files changed, 401 insertions(+), 48 deletions(-) create mode 100644 cli/src/prescale_cli/commands/db.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d87f63..085a77f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,14 @@ errors or Ctrl-C. Journeys that write change the copy, never the parent, and the load runs on the copy's own compute. Needs `NEON_API_KEY` and `NEON_PROJECT_ID`. The parent must be named, and the connection string is never printed, saved or uploaded; saved runs record - `config.database` (provider, parent, database name). `--db-name`, `--db-env` and + `config.database` (provider, parent, database name). `--db-name`, `--db-env` (repeatable) and `--db-pooled` cover multi-database branches, other variable names and Neon's pooler. + - Every well-known database variable that's set (`DIRECT_URL`, `POSTGRES_URL*`, + `DATABASE_URL_UNPOOLED`, `PG*`) is pointed at the copy too. + - If the app fails to start, its log is shown with the copy's URL and password masked, then + deleted. A cancelled CI job (SIGTERM) still deletes the copy. +- **`prescale db cleanup`** deletes PreScale's own leftover Neon copies (`prescale-...`) older + than six hours (`--older-than`), after you confirm. ## 1.2.0 — 2026-11-08 diff --git a/cli/src/prescale_cli/commands/db.py b/cli/src/prescale_cli/commands/db.py new file mode 100644 index 0000000..f0b6220 --- /dev/null +++ b/cli/src/prescale_cli/commands/db.py @@ -0,0 +1,53 @@ +"""`prescale db cleanup` - delete database copies a run couldn't clean up. + +Every `--db neon` run deletes its copy when it ends. A run that's killed outright +(a machine that dies, `kill -9`) can't, and Neon plans limit how many branches a +project has. This finds PreScale's own copies (`prescale-...`) older than a few +hours and deletes them. It never touches a branch PreScale didn't make. +""" + +from __future__ import annotations + +import click +from rich.console import Console + +from prescale_cli.dbbranch import DBBranchError, delete_copies, neon_settings, stale_copies + +console = Console() + + +@click.group() +def db() -> None: + """Database copies made by --db.""" + + +@db.command() +@click.option("--older-than", "older_than", default=6.0, type=float, show_default=True, + metavar="HOURS", help="Only copies created at least this many hours ago.") +@click.option("--yes", "yes", is_flag=True, help="Delete without asking.") +def cleanup(older_than: float, yes: bool) -> None: + """Delete leftover Neon copies (prescale-...) from runs that couldn't clean up. + + Needs NEON_API_KEY and NEON_PROJECT_ID, like --db neon. + """ + try: + api_key, project = neon_settings() + stale = stale_copies(api_key=api_key, project=project, older_than_hours=older_than) + except DBBranchError as exc: + console.print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) from None + if not stale: + console.print(f"[green]✓[/green] No PreScale copies older than {older_than:g}h.") + return + console.print(f"{len(stale)} PreScale cop{'y' if len(stale) == 1 else 'ies'} older than " + f"{older_than:g}h:") + for b in stale: + console.print(f" {b['name']} [dim](created {b.get('created_at')})[/dim]") + if not yes and not click.confirm("Delete them?", default=False): + raise SystemExit(0) + failed = delete_copies(stale, api_key=api_key, project=project) + for name, reason in failed: + console.print(f"[red]✗[/red] {name}: {reason}") + console.print(f"[green]✓[/green] Deleted {len(stale) - len(failed)}.") + if failed: + raise SystemExit(1) diff --git a/cli/src/prescale_cli/commands/replicaopts.py b/cli/src/prescale_cli/commands/replicaopts.py index 10e8088..30aa6e6 100644 --- a/cli/src/prescale_cli/commands/replicaopts.py +++ b/cli/src/prescale_cli/commands/replicaopts.py @@ -9,6 +9,10 @@ from __future__ import annotations import functools +import os +import signal +import sys +from urllib.parse import urlsplit import click from rich.console import Console @@ -20,6 +24,13 @@ # Where the --db choices and the copy's (non-secret) description live for the command. DB_META = "prescale.db" +# Database variables apps commonly read (Prisma, Vercel's Neon integration, libpq). +# Any of these that's set is pointed at the copy too, so no part of the app can +# reach the real database by another name. +_DIRECT_URL_VARS = ("DIRECT_URL", "DATABASE_URL_UNPOOLED", "POSTGRES_URL_NON_POOLING") +_URL_VARS = ("DATABASE_URL", "POSTGRES_URL", "POSTGRES_PRISMA_URL") + _DIRECT_URL_VARS +_PG_VARS = ("PGHOST", "PGHOST_UNPOOLED", "PGUSER", "PGPASSWORD", "PGDATABASE", "PGPORT") + app_option = click.option( "--app", "app", default=None, metavar="COMMAND", help='Start your app with COMMAND (e.g. "npm run build && npm start"), test it on the ' @@ -42,9 +53,11 @@ def db_options(fn): "default branch is often production, so PreScale never picks it.") @click.option("--db-name", "db_name", default=None, metavar="NAME", help="The database in the branch, if it has more than one.") - @click.option("--db-env", "db_env", default="DATABASE_URL", show_default=True, - metavar="VAR", help="The environment variable the app reads its database " - "URL from.") + @click.option("--db-env", "db_env", multiple=True, default=("DATABASE_URL",), + show_default=True, metavar="VAR", + help="An environment variable the app reads its database URL from " + "(repeatable). DIRECT_URL, POSTGRES_URL*, DATABASE_URL_UNPOOLED and " + "PG* are pointed at the copy too whenever they're set.") @click.option("--db-pooled", "db_pooled", is_flag=True, help="Give the app Neon's pooled connection URL (through its pooler).") @functools.wraps(fn) @@ -57,8 +70,14 @@ def wrapper(*args, db, db_parent, db_name, db_env, db_pooled, **kwargs): raise click.UsageError("--db neon needs --db-parent BRANCH, the branch to " "copy (e.g. staging).") click.get_current_context().meta[DB_META] = { - "provider": db, "parent": db_parent, "name": db_name, "env": db_env, + "provider": db, "parent": db_parent, "name": db_name, "env": list(db_env), "pooled": db_pooled} + # A cancelled CI job sends SIGTERM: turn it into a normal exit, so the + # app is stopped and the copy deleted on the way out. + try: + signal.signal(signal.SIGTERM, lambda *_: sys.exit(143)) + except (ValueError, OSError): # not the main thread, or no SIGTERM here + pass return fn(*args, **kwargs) return wrapper @@ -69,10 +88,11 @@ def _start_database(console: Console, quiet: bool) -> dict[str, str]: want = ctx.meta.get(DB_META) if not want: return {} + err = Console(stderr=True) if quiet else console # keep --json stdout clean try: api_key, project = neon_settings() except DBBranchError as exc: - console.print(f"[red]Error:[/red] {exc}") + err.print(f"[red]Error:[/red] {exc}") raise SystemExit(1) from None def left_behind(name, exc): @@ -88,14 +108,35 @@ def left_behind(name, exc): with console.status(f"Copying the Neon branch {want['parent']}…"): branch = ctx.with_resource(copy) except DBBranchError as exc: - console.print(f"[red]Error:[/red] {exc}") + err.print(f"[red]Error:[/red] {exc}") raise SystemExit(1) from None # What the Result may say about it: never the URL. want["copy"] = {"provider": "neon", "parent": branch.parent, "database": branch.database} + want["secrets"] = branch.secrets() + env = copy_environment(branch, want["env"], os.environ) if not quiet: console.print(f"[green]✓[/green] a throwaway copy of {branch.parent} is ready " f"([dim]{branch.name}[/dim]); it's deleted when the test ends") - return {want["env"]: branch.uri} + console.print(f" [dim]pointed at the copy: {', '.join(sorted(env))}[/dim]") + return env + + +def copy_environment(branch, named: list[str], current) -> dict[str, str]: + """The app's database variables, all pointing at the copy: the ones named + with --db-env, and every well-known one that's set (so the app can't reach + the real database through DIRECT_URL, POSTGRES_URL or PGHOST).""" + env = {name: branch.uri for name in named} + for name in _URL_VARS: + if name in current and name not in env: + env[name] = branch.direct_uri if name in _DIRECT_URL_VARS else branch.uri + if any(name in current for name in _PG_VARS): + parts = urlsplit(branch.direct_uri) + env.update({"PGHOST": parts.hostname or "", "PGUSER": parts.username or "", + "PGPASSWORD": parts.password or "", "PGDATABASE": parts.path.lstrip("/"), + "PGPORT": str(parts.port or 5432), "PGSSLMODE": "require"}) + if "PGHOST_UNPOOLED" in current: + env["PGHOST_UNPOOLED"] = parts.hostname or "" + return env def database_info() -> dict | None: @@ -114,6 +155,7 @@ def start_replica(app: str, url: str, *, timeout: float, console: Console, quiet console.print(f"[red]Error:[/red] {exc}") raise SystemExit(1) from None env = _start_database(console, quiet) + hidden = (click.get_current_context().meta.get(DB_META) or {}).get("secrets") if not quiet: console.print(Panel( f"Starting your app with [bold]{app}[/bold] and waiting for {url}.\n" + @@ -125,10 +167,10 @@ def start_replica(app: str, url: str, *, timeout: float, console: Console, quiet ctx = click.get_current_context() try: if quiet: - ctx.with_resource(replica(app, url, timeout=timeout, env=env)) + ctx.with_resource(replica(app, url, timeout=timeout, env=env, secrets=hidden)) else: with console.status("Building and starting your app…"): - ctx.with_resource(replica(app, url, timeout=timeout, env=env)) + ctx.with_resource(replica(app, url, timeout=timeout, env=env, secrets=hidden)) except ReplicaError as exc: console.print(f"[red]Error:[/red] {exc}") raise SystemExit(1) from None diff --git a/cli/src/prescale_cli/dbbranch.py b/cli/src/prescale_cli/dbbranch.py index 5c23d51..355aa03 100644 --- a/cli/src/prescale_cli/dbbranch.py +++ b/cli/src/prescale_cli/dbbranch.py @@ -21,13 +21,18 @@ from contextlib import contextmanager from dataclasses import dataclass, field from datetime import datetime, timezone +from urllib.parse import urlsplit import httpx NEON_API = "https://console.neon.tech/api/v2" READY_TIMEOUT = 120.0 +DELETE_TIMEOUT = 30.0 _POLL_S = 1.0 -_NAME_PREFIX = "prescale-" +NAME_PREFIX = "prescale-" +# Neon says 423 (another operation holds the resource) and 503 are safe to retry. +_RETRYABLE = (423, 503) +_RETRY_SCALE = 1.0 # tests set 0 to skip the backoff waits class DBBranchError(Exception): @@ -40,7 +45,16 @@ class Branch: name: str parent: str database: str - uri: str = field(repr=False) # a secret: never printed, saved or uploaded + # Secrets: never printed, saved or uploaded. + uri: str = field(repr=False) # direct, or pooled with --db-pooled + direct_uri: str = field(repr=False) + pooled_uri: str = field(repr=False) + + def secrets(self) -> set[str]: + """Every value that must never be shown: the URLs and the password.""" + values = {self.uri, self.direct_uri, self.pooled_uri} + values |= {urlsplit(u).password or "" for u in values} + return {v for v in values if v} def neon_settings(env: dict[str, str] | None = None) -> tuple[str, str]: @@ -61,7 +75,9 @@ def _call(client: httpx.Client, method: str, path: str, **kwargs) -> dict: if resp.status_code == 401: raise DBBranchError("Neon refused NEON_API_KEY (401). Check the key is current.") if resp.status_code == 404: - raise DBBranchError("Neon has no such project or branch (404). Check NEON_PROJECT_ID.") + what = ("project (check NEON_PROJECT_ID)" if path.count("/") <= 2 + else f"resource at {path.split('?')[0]}") + raise DBBranchError(f"Neon has no such {what} (404).") if resp.status_code >= 400: detail = "" try: @@ -73,17 +89,52 @@ def _call(client: httpx.Client, method: str, path: str, **kwargs) -> dict: return resp.json() if resp.content else {} +def _branches(client: httpx.Client, project: str, search: str | None = None) -> list[dict]: + """Every branch (following pagination), optionally narrowed by Neon's search.""" + out, cursor = [], None + for _ in range(50): # pages; a guard against a cursor that never ends + params = {k: v for k, v in {"search": search, "cursor": cursor}.items() if v} + page = _call(client, "GET", f"/projects/{project}/branches", params=params) + out += page.get("branches") or [] + cursor = (page.get("pagination") or {}).get("next") + if not cursor or not page.get("branches"): + break + return out + + def _find_parent(client: httpx.Client, project: str, parent: str) -> dict: - branches = _call(client, "GET", f"/projects/{project}/branches").get("branches") or [] - match = next((b for b in branches if parent in (b.get("name"), b.get("id"))), None) + # Search narrows by partial name or id; match exactly. + found = _branches(client, project, search=parent) + match = next((b for b in found if parent in (b.get("name"), b.get("id"))), None) if match is None: - names = ", ".join(sorted(b.get("name", "?") for b in branches)) or "none" - raise DBBranchError(f"No branch named {parent!r} in the Neon project. Branches: {names}.") + names = ", ".join(sorted(b.get("name", "?") for b in _branches(client, project))) + raise DBBranchError(f"No branch named {parent!r} in the Neon project. Branches: " + f"{names or 'none'}.") return match +def _delete(client: httpx.Client, project: str, branch_id: str, + timeout: float = DELETE_TIMEOUT) -> None: + """Delete a branch, retrying while Neon says it's locked or busy.""" + deadline, wait = time.monotonic() + timeout, 1.0 + while True: + try: + resp = client.request("DELETE", f"/projects/{project}/branches/{branch_id}") + except httpx.HTTPError as exc: + if time.monotonic() >= deadline: + raise DBBranchError(f"Couldn't reach Neon ({exc.__class__.__name__}).") from exc + else: + if resp.status_code in (200, 202, 204, 404): # 404: already gone + return + if resp.status_code not in _RETRYABLE or time.monotonic() >= deadline: + raise DBBranchError(f"Neon answered {resp.status_code} to the delete.") + time.sleep(min(wait, max(0.0, deadline - time.monotonic())) * _RETRY_SCALE) + wait *= 2 + + def _wait(client: httpx.Client, project: str, operations: list[dict], timeout: float) -> None: - pending = {op["id"] for op in operations if op.get("status") != "finished"} + pending = {op["id"] for op in operations + if op.get("status") not in ("finished", "skipped")} deadline = time.monotonic() + timeout while pending: if time.monotonic() > deadline: @@ -92,7 +143,7 @@ def _wait(client: httpx.Client, project: str, operations: list[dict], timeout: f op = _call(client, "GET", f"/projects/{project}/operations/{op_id}").get( "operation") or {} status = op.get("status") - if status == "finished": + if status in ("finished", "skipped"): # both are successful ends pending.discard(op_id) elif status in ("failed", "error", "cancelled"): raise DBBranchError(f"Neon couldn't create the branch (operation {status}).") @@ -128,11 +179,11 @@ def neon_branch(parent: str, *, api_key: str, project: str, database: str | None client = httpx.Client(base_url=NEON_API, timeout=30.0, transport=transport, headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"}) - created = None + created, name = None, None try: source = _find_parent(client, project, parent) stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") - name = f"{_NAME_PREFIX}{stamp}-{secrets.token_hex(3)}" + name = f"{NAME_PREFIX}{stamp}-{secrets.token_hex(3)}" made = _call(client, "POST", f"/projects/{project}/branches", json={ "branch": {"parent_id": source["id"], "name": name}, "endpoints": [{"type": "read_write"}], @@ -140,16 +191,68 @@ def neon_branch(parent: str, *, api_key: str, project: str, database: str | None created = made["branch"] _wait(client, project, made.get("operations") or [], timeout) db_name, role = _database(client, project, created["id"], database) - uri = _call(client, "GET", f"/projects/{project}/connection_uri", params={ - "branch_id": created["id"], "database_name": db_name, "role_name": role, - "pooled": "true" if pooled else "false"})["uri"] + + def uri_for(pool: bool) -> str: + return _call(client, "GET", f"/projects/{project}/connection_uri", params={ + "branch_id": created["id"], "database_name": db_name, "role_name": role, + "pooled": "true" if pool else "false"})["uri"] + direct, pooled_uri = uri_for(False), uri_for(True) yield Branch(id=created["id"], name=name, parent=source.get("name") or parent, - database=db_name, uri=uri) + database=db_name, uri=pooled_uri if pooled else direct, + direct_uri=direct, pooled_uri=pooled_uri) finally: - if created is not None: + try: + if created is None and name is not None: + # The create call failed, but Neon may have made the branch anyway + # (a timeout or a 5xx after the fact): find it by its unique name. + try: + created = next((b for b in _branches(client, project, search=name) + if b.get("name") == name), None) + except DBBranchError: + created = {"id": name, "name": name} # can't tell: say which to check + if created is not None: + try: + _delete(client, project, created["id"]) + except DBBranchError as exc: + if on_delete_failed: + on_delete_failed(created.get("name") or created["id"], exc) + finally: + client.close() + + +def stale_copies(*, api_key: str, project: str, older_than_hours: float, + transport: httpx.BaseTransport | None = None, + now: datetime | None = None) -> list[dict]: + """PreScale's own copies (`prescale-...`) created more than OLDER_THAN_HOURS + ago: what a run that couldn't clean up left behind.""" + now = now or datetime.now(timezone.utc) + with httpx.Client(base_url=NEON_API, timeout=30.0, transport=transport, + headers={"Authorization": f"Bearer {api_key}"}) as client: + found = _branches(client, project, search=NAME_PREFIX) + stale = [] + for b in found: + if not (b.get("name") or "").startswith(NAME_PREFIX): + continue + try: + created = datetime.fromisoformat(str(b.get("created_at")).replace("Z", "+00:00")) + except ValueError: + continue + if (now - created).total_seconds() >= older_than_hours * 3600: + stale.append(b) + return stale + + +def delete_copies(branches: list[dict], *, api_key: str, project: str, + transport: httpx.BaseTransport | None = None) -> list[tuple[str, str]]: + """Delete BRANCHES; the (name, reason) of each that couldn't be deleted.""" + failed = [] + with httpx.Client(base_url=NEON_API, timeout=30.0, transport=transport, + headers={"Authorization": f"Bearer {api_key}"}) as client: + for b in branches: + if not (b.get("name") or "").startswith(NAME_PREFIX): + continue # never anything PreScale didn't make try: - _call(client, "DELETE", f"/projects/{project}/branches/{created['id']}") + _delete(client, project, b["id"]) except DBBranchError as exc: - if on_delete_failed: - on_delete_failed(created.get("name") or created["id"], exc) - client.close() + failed.append((b.get("name") or b["id"], str(exc))) + return failed diff --git a/cli/src/prescale_cli/main.py b/cli/src/prescale_cli/main.py index 67f66a8..55a172f 100644 --- a/cli/src/prescale_cli/main.py +++ b/cli/src/prescale_cli/main.py @@ -7,6 +7,7 @@ audit, ci, compare, + db, history, investigate, journeys, @@ -59,6 +60,7 @@ def cli() -> None: cli.add_command(login.logout) cli.add_command(upload.upload) cli.add_command(ci.ci) +cli.add_command(db.db) def main() -> None: diff --git a/cli/src/prescale_cli/replica.py b/cli/src/prescale_cli/replica.py index e13554c..024c18f 100644 --- a/cli/src/prescale_cli/replica.py +++ b/cli/src/prescale_cli/replica.py @@ -52,21 +52,30 @@ def require_local(url: str) -> None: @contextmanager def replica(command: str, url: str, *, timeout: float = DEFAULT_TIMEOUT, cwd: str | None = None, - on_wait=None, env: dict[str, str] | None = None): + on_wait=None, env: dict[str, str] | None = None, secrets: set[str] | None = None): """Start `command`, wait for `url` to answer, yield, then stop the app and - everything it started (also on errors and Ctrl-C).""" + everything it started (also on errors and Ctrl-C). ENV is added to the app's + environment. SECRETS (e.g. a database copy's URL and password) are masked in + anything shown from the app's log, and the log is deleted when the app stops, + since apps often print their database URL.""" require_local(url) if _answers(url): raise ReplicaError(f"Something is already running at {url}. Stop it, or point the " "app at another port, so PreScale tests the build it starts.") log = tempfile.NamedTemporaryFile(prefix="prescale-app-", suffix=".log", delete=False) + hidden = {v for v in (secrets or set()) if v} process = _start(command, cwd, log, env) try: - _wait_until_ready(process, url, timeout, log.name, on_wait) + _wait_until_ready(process, url, timeout, log.name, on_wait, hidden) yield Replica(pid=process.pid, log_path=log.name) finally: _stop(process) log.close() + if hidden: + try: + os.unlink(log.name) + except OSError: + pass def _start(command: str, cwd: str | None, log, env: dict[str, str] | None = None @@ -91,22 +100,23 @@ def _answers(url: str) -> bool: def _wait_until_ready(process: subprocess.Popen, url: str, timeout: float, log_path: str, - on_wait) -> None: + on_wait, hidden: set[str] = frozenset()) -> None: + where = "" if hidden else f" ({log_path})" # deleted afterwards when it may hold secrets deadline = time.monotonic() + timeout started = time.monotonic() while time.monotonic() < deadline: code = process.poll() if code is not None: raise ReplicaError(f"The app exited (code {code}) before {url} answered. " - f"The end of its log ({log_path}):\n{_tail(log_path)}") + f"The end of its log{where}:\n{_tail(log_path, hidden)}") if _answers(url): return if on_wait: on_wait(time.monotonic() - started) time.sleep(_POLL_S) raise ReplicaError(f"The app didn't answer at {url} within {timeout:g}s. Raise " - f"--app-timeout if the build takes longer. The end of its log " - f"({log_path}):\n{_tail(log_path)}") + f"--app-timeout if the build takes longer. The end of its log" + f"{where}:\n{_tail(log_path, hidden)}") def _stop(process: subprocess.Popen) -> None: @@ -146,10 +156,17 @@ def _group_alive(pgid: int) -> bool: return True -def _tail(path: str) -> str: +def mask(text: str, hidden: set[str]) -> str: + """TEXT with every secret in HIDDEN replaced by ***, longest first.""" + for value in sorted(hidden, key=len, reverse=True): + text = text.replace(value, "***") + return text + + +def _tail(path: str, hidden: set[str] = frozenset()) -> str: try: with open(path, encoding="utf-8", errors="replace") as fh: lines = fh.read().splitlines() except OSError: return "(no log)" - return "\n".join(lines[-_LOG_LINES:]) or "(the app printed nothing)" + return mask("\n".join(lines[-_LOG_LINES:]), hidden) or "(the app printed nothing)" diff --git a/cli/tests/test_dbbranch.py b/cli/tests/test_dbbranch.py index 503f50c..b443b62 100644 --- a/cli/tests/test_dbbranch.py +++ b/cli/tests/test_dbbranch.py @@ -3,6 +3,8 @@ import functools import json import sys +import tempfile +from pathlib import Path import httpx import pytest @@ -13,29 +15,47 @@ from prescale_cli.main import cli URI = "postgresql://owner:s3cr3t-pw@ep-copy-123.us-east-2.aws.neon.tech/shop?sslmode=require" +POOLED = ("postgresql://owner:s3cr3t-pw@ep-copy-123-pooler.us-east-2.aws.neon.tech/shop" + "?sslmode=require") class FakeNeon: """Enough of Neon's API: branches, operations, databases, connection URI.""" def __init__(self, *, databases=("shop",), op_status=("running", "finished"), - branches=("main", "staging")): + branches=("main", "staging"), page_size=100, delete_locked=0, + post_fails_after_create=False): self.calls: list[tuple[str, str]] = [] self.created: dict | None = None self.deleted: list[str] = [] self.databases = databases self.op_status = list(op_status) - self.branches = [{"id": f"br-{n}", "name": n} for n in branches] + self.branches = [{"id": f"br-{n}", "name": n, "created_at": "2026-11-01T00:00:00Z"} + for n in branches] + self.page_size = page_size + self.delete_locked = delete_locked + self.post_fails_after_create = post_fails_after_create def handler(self, request: httpx.Request) -> httpx.Response: path, method = request.url.path.removeprefix("/api/v2"), request.method self.calls.append((method, path)) assert request.headers["authorization"] == "Bearer key-123" if method == "GET" and path == "/projects/p1/branches": - return httpx.Response(200, json={"branches": self.branches}) + search = request.url.params.get("search", "") + found = [b for b in self.branches if search in b["name"] or search in b["id"]] + start = int(request.url.params.get("cursor", "0")) + page = found[start:start + self.page_size] + more = start + self.page_size < len(found) + return httpx.Response(200, json={ + "branches": page, + "pagination": {"next": str(start + self.page_size)} if more else {}}) if method == "POST" and path == "/projects/p1/branches": body = json.loads(request.content) self.created = body + self.branches.append({"id": "br-copy", "name": body["branch"]["name"], + "created_at": "2026-11-13T00:00:00Z"}) + if self.post_fails_after_create: + return httpx.Response(500, json={"message": "oops"}) return httpx.Response(201, json={ "branch": {"id": "br-copy", "name": body["branch"]["name"]}, "operations": [{"id": "op1", "status": "running"}]}) @@ -48,10 +68,16 @@ def handler(self, request: httpx.Request) -> httpx.Response: if method == "GET" and path == "/projects/p1/connection_uri": assert request.url.params["branch_id"] == "br-copy" assert request.url.params["role_name"] == "owner" + pooled = request.url.params["pooled"] == "true" self.uri_params = dict(request.url.params) - return httpx.Response(200, json={"uri": URI}) - if method == "DELETE" and path == "/projects/p1/branches/br-copy": - self.deleted.append("br-copy") + return httpx.Response(200, json={"uri": POOLED if pooled else URI}) + if method == "DELETE" and path.startswith("/projects/p1/branches/"): + if self.delete_locked: + self.delete_locked -= 1 + return httpx.Response(423, json={"message": "locked"}) + branch = path.rsplit("/", 1)[1] + self.deleted.append(branch) + self.branches = [b for b in self.branches if b["id"] != branch] return httpx.Response(200, json={}) return httpx.Response(404, json={"message": "nope"}) @@ -62,6 +88,7 @@ def transport(self): @pytest.fixture(autouse=True) def fast(monkeypatch): monkeypatch.setattr(dbbranch, "_POLL_S", 0) + monkeypatch.setattr(dbbranch, "_RETRY_SCALE", 0) def _branch(fake, parent="staging", **kw): @@ -79,7 +106,8 @@ def test_copies_the_named_branch_waits_until_ready_and_deletes_it(): assert body["branch"]["name"].startswith("prescale-") assert body["endpoints"] == [{"type": "read_write"}] assert fake.calls.count(("GET", "/projects/p1/operations/op1")) == 2 # polled until finished - assert fake.uri_params["pooled"] == "false" + assert branch.uri == URI and branch.direct_uri == URI and branch.pooled_uri == POOLED + assert {URI, POOLED, "s3cr3t-pw"} <= branch.secrets() assert fake.deleted == ["br-copy"] assert URI not in repr(branch) and "s3cr3t" not in repr(branch) @@ -112,8 +140,7 @@ def test_several_databases_need_a_name_and_pooled_is_passed(): pass fake = FakeNeon(databases=("shop", "analytics")) with _branch(fake, database="analytics", pooled=True) as branch: - assert branch.database == "analytics" - assert fake.uri_params["pooled"] == "true" + assert branch.database == "analytics" and branch.uri == POOLED def test_a_refused_key_is_named(): @@ -152,6 +179,9 @@ def test_missing_neon_settings_stop_before_the_app_starts(monkeypatch, tmp_path) APP = """ import http.server, os, sys open(sys.argv[2], "w").write(os.environ.get("DATABASE_URL", "")) +print("connecting to", os.environ.get("DATABASE_URL"), flush=True) +if len(sys.argv) > 3: + sys.exit(3) # a startup failure that printed the URL class H(http.server.BaseHTTPRequestHandler): def do_GET(self): self.send_response(200); self.send_header("Content-Length", "2"); self.end_headers() @@ -189,3 +219,86 @@ def test_run_gives_the_app_the_copy_and_never_shows_its_url(monkeypatch, tmp_pat saved = "".join(p.read_text() for p in store.rglob("*.json")) for text in (res.output, saved): assert "s3cr3t" not in text and "ep-copy-123" not in text + + +def test_skipped_operations_count_as_done(): + fake = FakeNeon(op_status=("skipped",)) + with _branch(fake) as branch: + assert branch.uri == URI + + +def test_the_parent_is_found_past_the_first_page(): + fake = FakeNeon(branches=[f"preview-{i}" for i in range(30)] + ["staging"], page_size=10) + with _branch(fake) as branch: + assert branch.parent == "staging" + assert fake.created["branch"]["parent_id"] == "br-staging" + + +def test_a_create_that_failed_after_neon_made_the_branch_still_cleans_up(): + fake = FakeNeon(post_fails_after_create=True) + with pytest.raises(DBBranchError, match="500"), _branch(fake): + pass + assert fake.deleted == ["br-copy"] + + +def test_a_locked_delete_is_retried(): + fake = FakeNeon(delete_locked=2) + with _branch(fake): + pass + assert fake.deleted == ["br-copy"] + + +def test_every_known_database_variable_points_at_the_copy(): + from prescale_cli.commands.replicaopts import copy_environment + fake = FakeNeon() + with _branch(fake) as branch: + env = copy_environment(branch, ["DATABASE_URL", "APP_DB"], { + "DIRECT_URL": "postgres://staging", "POSTGRES_PRISMA_URL": "postgres://staging", + "PGHOST": "staging.example.com", "PGPASSWORD": "real", "HOME": "/root"}) + assert env["DATABASE_URL"] == env["APP_DB"] == env["POSTGRES_PRISMA_URL"] == URI + assert env["DIRECT_URL"] == URI + assert env["PGHOST"] == "ep-copy-123.us-east-2.aws.neon.tech" + assert env["PGPASSWORD"] == "s3cr3t-pw" and env["PGDATABASE"] == "shop" + assert "HOME" not in env + + +def test_cleanup_finds_only_old_prescale_copies_and_deletes_them(monkeypatch): + from datetime import datetime, timezone + fake = FakeNeon(branches=("main", "staging")) + fake.branches += [ + {"id": "br-old", "name": "prescale-20261101-000000-aaaaaa", + "created_at": "2026-11-01T00:00:00Z"}, + {"id": "br-new", "name": "prescale-20261113-000000-bbbbbb", + "created_at": "2026-11-13T09:00:00Z"}, + {"id": "br-x", "name": "prescale-lookalike-but-old-and-not-ours", + "created_at": "2026-10-01T00:00:00Z"}, + ] + now = datetime(2026, 11, 13, 10, 0, tzinfo=timezone.utc) + stale = dbbranch.stale_copies(api_key="key-123", project="p1", older_than_hours=6, + transport=fake.transport(), now=now) + assert {b["id"] for b in stale} == {"br-old", "br-x"} # the prefix is ours + assert dbbranch.delete_copies(stale, api_key="key-123", project="p1", + transport=fake.transport()) == [] + assert sorted(fake.deleted) == ["br-old", "br-x"] + assert "br-staging" not in fake.deleted and "br-main" not in fake.deleted + + +def test_an_app_that_prints_its_url_and_dies_never_shows_the_secret(monkeypatch, tmp_path): + fake = FakeNeon() + from prescale_cli.commands import replicaopts + monkeypatch.setattr(replicaopts, "neon_branch", + functools.partial(neon_branch, transport=fake.transport())) + monkeypatch.setenv("NEON_API_KEY", "key-123") + monkeypatch.setenv("NEON_PROJECT_ID", "p1") + (tmp_path / "app.py").write_text(APP) + res = CliRunner().invoke(cli, [ + "run", "http://127.0.0.1:9", "--urls", + "--app", f"{sys.executable} {tmp_path / 'app.py'} 9 {tmp_path / 'seen'} die", + "--db", "neon", "--db-parent", "staging"]) + assert res.exit_code == 1 + flat = "".join(res.output.split()) # Rich wraps long lines + assert "connectingto***" in flat # the log tail is shown, masked + assert "s3cr3t" not in flat and "ep-copy-123" not in flat + assert fake.deleted == ["br-copy"] + leftover = list(Path(tempfile.gettempdir()).glob("prescale-app-*.log")) + assert not any("s3cr3t" in p.read_text(errors="replace") for p in leftover) diff --git a/docs/guides/replica-mode.mdx b/docs/guides/replica-mode.mdx index c00e1f7..bc14dc9 100644 --- a/docs/guides/replica-mode.mdx +++ b/docs/guides/replica-mode.mdx @@ -67,9 +67,26 @@ prescale run http://localhost:3000 --app "npm run build && npm start" \ `DATABASE_URL`, or the variable you name with `--db-env`), never into the terminal, the saved run or an upload. +- **Every database variable points at the copy.** Besides `DATABASE_URL` (or the variables you + name with `--db-env`, repeatable), any of `DIRECT_URL`, `POSTGRES_URL`, `POSTGRES_PRISMA_URL`, + `POSTGRES_URL_NON_POOLING`, `DATABASE_URL_UNPOOLED` and the `PG*` variables that's set is + pointed at the copy too, so no part of the app reaches the real database by another name. + PreScale lists the ones it replaced. +- **Secrets stay out of logs.** If the app fails to start, the end of its log is shown with the + copy's URL and password replaced by `***`, and the log file is deleted. +- **Cancelled CI jobs clean up too.** A SIGTERM (a cancelled GitHub Actions job) stops the app + and deletes the copy on the way out. + Options: `--db-name` picks the database when the branch has more than one, and `--db-pooled` gives the app Neon's pooled URL. +### Leftover copies + +A run that's killed outright (a machine that dies, `kill -9`) can't delete its copy, and Neon +plans limit how many branches a project has. `prescale db cleanup` lists PreScale's own copies +(`prescale-...`) older than six hours (`--older-than HOURS`) and deletes them after you confirm +(`--yes` to skip). It never touches a branch PreScale didn't make. + ## In CI, on every pull request In CI, replica mode needs no deploy at all: the [GitHub Action](/guides/github-action) From 842e9bdd7ab2279ab16b19c117b1b575a04bfc36 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 18:32:13 +0000 Subject: [PATCH 3/4] fix(replica): copy wins over .env files, signals can't interrupt cleanup, cleanup matches only our branches --- CHANGELOG.md | 5 +- cli/src/prescale_cli/commands/replicaopts.py | 76 +++++++++++++++----- cli/src/prescale_cli/dbbranch.py | 19 +++-- cli/src/prescale_cli/replica.py | 9 ++- cli/tests/test_dbbranch.py | 63 +++++++++++++++- docs/guides/replica-mode.mdx | 16 +++-- 6 files changed, 153 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 085a77f..8cea907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,9 @@ and the connection string is never printed, saved or uploaded; saved runs record `config.database` (provider, parent, database name). `--db-name`, `--db-env` (repeatable) and `--db-pooled` cover multi-database branches, other variable names and Neon's pooler. - - Every well-known database variable that's set (`DIRECT_URL`, `POSTGRES_URL*`, - `DATABASE_URL_UNPOOLED`, `PG*`) is pointed at the copy too. + - Every well-known database variable (`DIRECT_URL`, `POSTGRES_URL*`, `DATABASE_URL_UNPOOLED`, + `PG*`) is set to the copy, even when unset, so a `.env` file can't point the app elsewhere; + other variables holding a Postgres URL get a warning. - If the app fails to start, its log is shown with the copy's URL and password masked, then deleted. A cancelled CI job (SIGTERM) still deletes the copy. - **`prescale db cleanup`** deletes PreScale's own leftover Neon copies (`prescale-...`) older diff --git a/cli/src/prescale_cli/commands/replicaopts.py b/cli/src/prescale_cli/commands/replicaopts.py index 30aa6e6..8ab6cac 100644 --- a/cli/src/prescale_cli/commands/replicaopts.py +++ b/cli/src/prescale_cli/commands/replicaopts.py @@ -72,16 +72,48 @@ def wrapper(*args, db, db_parent, db_name, db_env, db_pooled, **kwargs): click.get_current_context().meta[DB_META] = { "provider": db, "parent": db_parent, "name": db_name, "env": list(db_env), "pooled": db_pooled} - # A cancelled CI job sends SIGTERM: turn it into a normal exit, so the - # app is stopped and the copy deleted on the way out. - try: - signal.signal(signal.SIGTERM, lambda *_: sys.exit(143)) - except (ValueError, OSError): # not the main thread, or no SIGTERM here - pass + _guard_signals(click.get_current_context()) return fn(*args, **kwargs) return wrapper +def _guard_signals(ctx: click.Context) -> None: + """A cancelled CI job sends SIGINT, then SIGTERM seconds later, then kills. + The first signal ends the run normally (so the app stops and the copy is + deleted), the app gets a short stop, and later signals are ignored so they + can't interrupt the delete. The old handlers come back afterwards.""" + from prescale_cli import replica as replica_mod + + def first(signum, _frame): + for sig in _SIGNALS: + try: + signal.signal(sig, signal.SIG_IGN) + except (ValueError, OSError): + pass + replica_mod.QUICK_STOP = True + if signum == signal.SIGINT: + raise KeyboardInterrupt + sys.exit(128 + signum) + + try: + previous = {sig: signal.signal(sig, first) for sig in _SIGNALS} + except (ValueError, OSError): # not the main thread, or no such signal here + return + + def restore(): + replica_mod.QUICK_STOP = False + for sig, handler in previous.items(): + try: + signal.signal(sig, handler) + except (ValueError, OSError, TypeError): + pass + ctx.call_on_close(restore) + + +_SIGNALS = tuple(s for s in (getattr(signal, "SIGINT", None), getattr(signal, "SIGTERM", None)) + if s is not None) + + def _start_database(console: Console, quiet: bool) -> dict[str, str]: """Make the database copy if --db asked for one; the app's extra environment.""" ctx = click.get_current_context() @@ -118,27 +150,37 @@ def left_behind(name, exc): console.print(f"[green]✓[/green] a throwaway copy of {branch.parent} is ready " f"([dim]{branch.name}[/dim]); it's deleted when the test ends") console.print(f" [dim]pointed at the copy: {', '.join(sorted(env))}[/dim]") + others = other_databases(env, os.environ) + if others: + Console(stderr=True).print( + f"[yellow]⚠ {', '.join(others)} also hold a Postgres URL and still point where " + "they did. If the app reads them, add them with --db-env.[/yellow]") return env def copy_environment(branch, named: list[str], current) -> dict[str, str]: """The app's database variables, all pointing at the copy: the ones named - with --db-env, and every well-known one that's set (so the app can't reach - the real database through DIRECT_URL, POSTGRES_URL or PGHOST).""" + with --db-env, and every well-known one, set or not. Always set, because + apps often load these from a .env file, and dotenv never overrides a + variable that's already set: so the copy wins over the file.""" env = {name: branch.uri for name in named} for name in _URL_VARS: - if name in current and name not in env: - env[name] = branch.direct_uri if name in _DIRECT_URL_VARS else branch.uri - if any(name in current for name in _PG_VARS): - parts = urlsplit(branch.direct_uri) - env.update({"PGHOST": parts.hostname or "", "PGUSER": parts.username or "", - "PGPASSWORD": parts.password or "", "PGDATABASE": parts.path.lstrip("/"), - "PGPORT": str(parts.port or 5432), "PGSSLMODE": "require"}) - if "PGHOST_UNPOOLED" in current: - env["PGHOST_UNPOOLED"] = parts.hostname or "" + env.setdefault(name, branch.direct_uri if name in _DIRECT_URL_VARS else branch.uri) + parts = urlsplit(branch.direct_uri) + env.update({"PGHOST": parts.hostname or "", "PGHOST_UNPOOLED": parts.hostname or "", + "PGUSER": parts.username or "", "PGPASSWORD": parts.password or "", + "PGDATABASE": parts.path.lstrip("/"), "PGPORT": str(parts.port or 5432), + "PGSSLMODE": "require"}) return env +def other_databases(env: dict[str, str], current) -> list[str]: + """Names of variables (not pointed at the copy) that hold a Postgres URL: the + app might still reach a real database through them.""" + return sorted(name for name, value in current.items() + if name not in env and str(value).startswith(("postgres://", "postgresql://"))) + + def database_info() -> dict | None: """The database copy this command's app ran against, for the Result.""" ctx = click.get_current_context(silent=True) diff --git a/cli/src/prescale_cli/dbbranch.py b/cli/src/prescale_cli/dbbranch.py index 355aa03..8504a76 100644 --- a/cli/src/prescale_cli/dbbranch.py +++ b/cli/src/prescale_cli/dbbranch.py @@ -16,6 +16,7 @@ from __future__ import annotations import os +import re import secrets import time from contextlib import contextmanager @@ -30,6 +31,9 @@ DELETE_TIMEOUT = 30.0 _POLL_S = 1.0 NAME_PREFIX = "prescale-" +# Exactly the names neon_branch generates, so cleanup never matches a user's +# own branch that happens to start with "prescale-". +OUR_NAME = re.compile(r"^prescale-\d{8}-\d{6}-[0-9a-f]{6}$") # Neon says 423 (another operation holds the resource) and 503 are safe to retry. _RETRYABLE = (423, 503) _RETRY_SCALE = 1.0 # tests set 0 to skip the backoff waits @@ -208,8 +212,10 @@ def uri_for(pool: bool) -> str: try: created = next((b for b in _branches(client, project, search=name) if b.get("name") == name), None) - except DBBranchError: - created = {"id": name, "name": name} # can't tell: say which to check + except DBBranchError as exc: + # Can't tell whether it exists: say which branch to check. + if on_delete_failed: + on_delete_failed(name, exc) if created is not None: try: _delete(client, project, created["id"]) @@ -220,6 +226,11 @@ def uri_for(pool: bool) -> str: client.close() +def _ours(branch: dict) -> bool: + return (bool(OUR_NAME.match(branch.get("name") or "")) and not branch.get("default") + and not branch.get("protected") and not branch.get("primary")) + + def stale_copies(*, api_key: str, project: str, older_than_hours: float, transport: httpx.BaseTransport | None = None, now: datetime | None = None) -> list[dict]: @@ -231,7 +242,7 @@ def stale_copies(*, api_key: str, project: str, older_than_hours: float, found = _branches(client, project, search=NAME_PREFIX) stale = [] for b in found: - if not (b.get("name") or "").startswith(NAME_PREFIX): + if not _ours(b): continue try: created = datetime.fromisoformat(str(b.get("created_at")).replace("Z", "+00:00")) @@ -249,7 +260,7 @@ def delete_copies(branches: list[dict], *, api_key: str, project: str, with httpx.Client(base_url=NEON_API, timeout=30.0, transport=transport, headers={"Authorization": f"Bearer {api_key}"}) as client: for b in branches: - if not (b.get("name") or "").startswith(NAME_PREFIX): + if not _ours(b): continue # never anything PreScale didn't make try: _delete(client, project, b["id"]) diff --git a/cli/src/prescale_cli/replica.py b/cli/src/prescale_cli/replica.py index 024c18f..441fd25 100644 --- a/cli/src/prescale_cli/replica.py +++ b/cli/src/prescale_cli/replica.py @@ -30,6 +30,10 @@ _POLL_S = 0.5 _STOP_GRACE_S = 10.0 +_QUICK_STOP_S = 2.0 +# Set when the run is being cancelled (--db's signal guard): stop the app fast, so +# its database copy can still be deleted before the job is killed. +QUICK_STOP = False _LOG_LINES = 15 @@ -131,7 +135,8 @@ def _stop(process: subprocess.Popen) -> None: os.killpg(process.pid, signal.SIGTERM) except ProcessLookupError: return - deadline = time.monotonic() + _STOP_GRACE_S + grace = _QUICK_STOP_S if QUICK_STOP else _STOP_GRACE_S + deadline = time.monotonic() + grace while time.monotonic() < deadline and _group_alive(process.pid): process.poll() # reap the leader so it doesn't linger as a zombie time.sleep(0.1) @@ -141,7 +146,7 @@ def _stop(process: subprocess.Popen) -> None: except ProcessLookupError: pass try: - process.wait(timeout=_STOP_GRACE_S) + process.wait(timeout=grace) except subprocess.TimeoutExpired: pass diff --git a/cli/tests/test_dbbranch.py b/cli/tests/test_dbbranch.py index b443b62..7751ca9 100644 --- a/cli/tests/test_dbbranch.py +++ b/cli/tests/test_dbbranch.py @@ -260,6 +260,17 @@ def test_every_known_database_variable_points_at_the_copy(): assert env["PGHOST"] == "ep-copy-123.us-east-2.aws.neon.tech" assert env["PGPASSWORD"] == "s3cr3t-pw" and env["PGDATABASE"] == "shop" assert "HOME" not in env + # set even when unset here: an app's .env file can't win (dotenv never overrides) + empty = copy_environment(branch, ["DATABASE_URL"], {}) + assert empty["POSTGRES_URL"] == URI and empty["DIRECT_URL"] == URI + assert empty["PGHOST"] == "ep-copy-123.us-east-2.aws.neon.tech" + + +def test_other_postgres_urls_are_named(): + from prescale_cli.commands.replicaopts import other_databases + assert other_databases({"DATABASE_URL": URI}, { + "DATABASE_URL": "postgres://staging", "ANALYTICS_DB": "postgresql://warehouse/x", + "HOME": "/root"}) == ["ANALYTICS_DB"] def test_cleanup_finds_only_old_prescale_copies_and_deletes_them(monkeypatch): @@ -270,16 +281,23 @@ def test_cleanup_finds_only_old_prescale_copies_and_deletes_them(monkeypatch): "created_at": "2026-11-01T00:00:00Z"}, {"id": "br-new", "name": "prescale-20261113-000000-bbbbbb", "created_at": "2026-11-13T09:00:00Z"}, - {"id": "br-x", "name": "prescale-lookalike-but-old-and-not-ours", + {"id": "br-x", "name": "prescale-demo", # a user's own branch + "created_at": "2026-10-01T00:00:00Z"}, + {"id": "br-p", "name": "prescale-20261001-000000-cccccc", "protected": True, "created_at": "2026-10-01T00:00:00Z"}, ] now = datetime(2026, 11, 13, 10, 0, tzinfo=timezone.utc) stale = dbbranch.stale_copies(api_key="key-123", project="p1", older_than_hours=6, transport=fake.transport(), now=now) - assert {b["id"] for b in stale} == {"br-old", "br-x"} # the prefix is ours + assert {b["id"] for b in stale} == {"br-old"} # only names PreScale generates assert dbbranch.delete_copies(stale, api_key="key-123", project="p1", transport=fake.transport()) == [] - assert sorted(fake.deleted) == ["br-old", "br-x"] + assert fake.deleted == ["br-old"] + # and delete_copies itself refuses branches PreScale didn't make, even if handed them + not_ours = [b for b in fake.branches if b["id"] in ("br-main", "br-staging", "br-x", "br-p")] + assert dbbranch.delete_copies(not_ours, api_key="key-123", project="p1", + transport=fake.transport()) == [] + assert fake.deleted == ["br-old"] assert "br-staging" not in fake.deleted and "br-main" not in fake.deleted @@ -302,3 +320,42 @@ def test_an_app_that_prints_its_url_and_dies_never_shows_the_secret(monkeypatch, assert fake.deleted == ["br-copy"] leftover = list(Path(tempfile.gettempdir()).glob("prescale-app-*.log")) assert not any("s3cr3t" in p.read_text(errors="replace") for p in leftover) + + +def test_when_a_failed_create_cant_be_checked_the_user_is_told(): + fake = FakeNeon(post_fails_after_create=True) + calls = {"n": 0} + + def flaky(request): + if request.method == "GET" and "search=prescale-" in str(request.url): + calls["n"] += 1 + return httpx.Response(500, json={"message": "down"}) + return fake.handler(request) + told = [] + with pytest.raises(DBBranchError), neon_branch( + "staging", api_key="key-123", project="p1", transport=httpx.MockTransport(flaky), + on_delete_failed=lambda name, exc: told.append(name)): + pass + assert calls["n"] == 1 and told and told[0].startswith("prescale-") + + +def test_the_first_signal_ends_the_run_and_later_ones_cant_interrupt_cleanup(): + import signal + + import click + + from prescale_cli import replica as replica_mod + from prescale_cli.commands.replicaopts import _guard_signals + before = signal.getsignal(signal.SIGTERM) + with click.Context(click.Command("x")) as ctx: + _guard_signals(ctx) + handler = signal.getsignal(signal.SIGTERM) + assert handler not in (before, signal.SIG_DFL) + with pytest.raises(SystemExit) as exit_: + handler(signal.SIGTERM, None) + assert exit_.value.code == 128 + signal.SIGTERM + assert replica_mod.QUICK_STOP is True # the app gets a short stop + assert signal.getsignal(signal.SIGTERM) == signal.SIG_IGN + assert signal.getsignal(signal.SIGINT) == signal.SIG_IGN # a 2nd Ctrl-C too + assert signal.getsignal(signal.SIGTERM) == before # restored afterwards + assert replica_mod.QUICK_STOP is False diff --git a/docs/guides/replica-mode.mdx b/docs/guides/replica-mode.mdx index bc14dc9..fc49803 100644 --- a/docs/guides/replica-mode.mdx +++ b/docs/guides/replica-mode.mdx @@ -68,14 +68,16 @@ prescale run http://localhost:3000 --app "npm run build && npm start" \ saved run or an upload. - **Every database variable points at the copy.** Besides `DATABASE_URL` (or the variables you - name with `--db-env`, repeatable), any of `DIRECT_URL`, `POSTGRES_URL`, `POSTGRES_PRISMA_URL`, - `POSTGRES_URL_NON_POOLING`, `DATABASE_URL_UNPOOLED` and the `PG*` variables that's set is - pointed at the copy too, so no part of the app reaches the real database by another name. - PreScale lists the ones it replaced. + name with `--db-env`, repeatable), PreScale always sets `DIRECT_URL`, `POSTGRES_URL`, + `POSTGRES_PRISMA_URL`, `POSTGRES_URL_NON_POOLING`, `DATABASE_URL_UNPOOLED` and the `PG*` + variables to the copy, even if they aren't set. Apps often load these from a `.env` file, + and dotenv never overrides a variable that's already set, so the copy wins over the file. + Any other variable holding a Postgres URL gets a warning: add it with `--db-env`. - **Secrets stay out of logs.** If the app fails to start, the end of its log is shown with the copy's URL and password replaced by `***`, and the log file is deleted. -- **Cancelled CI jobs clean up too.** A SIGTERM (a cancelled GitHub Actions job) stops the app - and deletes the copy on the way out. +- **Cancelled CI jobs clean up too.** The first Ctrl-C or SIGTERM (a cancelled GitHub Actions + job) stops the app quickly and deletes the copy; further signals are ignored so they can't + interrupt the delete. Options: `--db-name` picks the database when the branch has more than one, and `--db-pooled` gives the app Neon's pooled URL. @@ -84,7 +86,7 @@ gives the app Neon's pooled URL. A run that's killed outright (a machine that dies, `kill -9`) can't delete its copy, and Neon plans limit how many branches a project has. `prescale db cleanup` lists PreScale's own copies -(`prescale-...`) older than six hours (`--older-than HOURS`) and deletes them after you confirm +(named `prescale--